Added base for version_scraper

This commit is contained in:
Emily 2018-05-24 12:57:38 +02:00
parent 4934a269a9
commit 9d6d544e0a
2 changed files with 34 additions and 0 deletions

View File

@ -3,6 +3,9 @@
"last": 0, "last": 0,
"failed": [] "failed": []
}, },
"version_scrape": {
"last": 0
},
"downloader": { "downloader": {
"last": 0, "last": 0,
"failed": [] "failed": []

31
version_scrape.py Normal file
View File

@ -0,0 +1,31 @@
#Used for scraping osu version names from osu.ppy.sh and storing them into self database
import urllib.request, json
import MySQLdb
import MySQLdb.cursors
import time, calendar
import atexit
with open("config.json", "r") as f:
config = json.load(f)
with open("memory.json", "r") as f:
memory = json.load(f)
sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
cur = sql.cursor()
def on_close():
with open("memory.json", "w") as f:
json.dump(memory, f)
print("Closing...")
atexit.register(on_close)
URL = "https://osu.ppy.sh/web/get-internal-version.php?v={}"
VERSIONS = urllib.request.urlopen(URL.format("_")) # Gets all versions
BEGINNING = memory["version_scrape"]["last"]
for i in range(VERSIONS):
pass #Unfinished