Sunpy/osu-wayback
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Finished version_scrape script

Browse Source
This commit is contained in:
Emily 2018-05-29 10:46:42 +02:00
parent 92ab017689
commit 3e9a7d05d7
1 changed files with 48 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
version_scrape.py
View File

@ -1,11 +1,14 @@
#Used for scraping osu version names from osu.ppy.sh and storing them into self database #Used for scraping osu version names from osu.ppy.sh and storing them into self database
import urllib.request, json import json
import MySQLdb import MySQLdb
import MySQLdb.cursors import MySQLdb.cursors
import time, calendar, string import time, calendar, string
import atexit import atexit
# Because ppy site is dumb
import requests
with open("config.json", "r") as f: with open("config.json", "r") as f:
config = json.load(f) config = json.load(f)
@ -25,69 +28,54 @@ atexit.register(on_close)
#--------------------------------------- #---------------------------------------
VERSIONS = check("_") # Gets all versions SCAN = "_" + string.printable[:62] + "-+.:,"
BRANCH = { NAMES = []
"_": VERSIONS
}
SCAN = string.printable[:62] + "-+.:,"
def check(s): def check(s):
return urllib.request.urlopen("https://osu.ppy.sh/web/get-internal-version.php?v={}".format(s)).read().decode() o = requests.get("https://osu.ppy.sh/web/get-internal-version.php?v={}".format(s))
i = int(o.content.decode())
return i
""" def scan_line(s):
TMP = "" global NAMES
def scope(v): data = {
global TMP "scan": [],
TMP = "" "items": 0
o_key = "" }
o_value = v["_"] items_found = 0
v = _scope(v) current_string = s + "_"
print(v) for i in range(1, len(SCAN)):
print(TMP) r = check(current_string)
print(o_key)
print(o_value)
def _scope(v): if current_string.endswith("_"):
global TMP data["items"] = r
for key, value in v.items(): if r is 0: # We hit the end! Add it to the entries list
if type(value) is int: c = current_string[:-1]
if len(v) is 1: print("FOUND: {}".format(c))
return key cur.execute("INSERT INTO osu_builds (id,version) VALUES (NULL,%s)", [ c ])
continue sql.commit()
TMP += key NAMES.append(c)
return _scope(v[key]) break
elif r > 0: # This is not the length scan and an entry was found so... we add it :D
print("Pending branch: {} ({} found)".format(current_string, r))
data["scan"].append(current_string)
items_found += r
if items_found >= data["items"]: # We have all the entries for this branch
print("Found all items in branch: {} ({} found)".format(current_string[:-1], items_found))
break
return {o_key: o_value} current_string = current_string[:-1] + SCAN[i]
return data
OUT = "" def branch(s):
ENTRIES = v["_"] scan_data = scan_line(s)
for key, value in v.items(): for item in scan_data["scan"]:
if key is "_": branch(item)
continue
elif type(value) is int:
return key
else:
S = scope(v[key])
OUT += key + S.keys()[0]
ENTRIES = S.values()[0]
return {OUT: ENTRIES}
"""
char_index = 0 branch("") # LEL INIT
current_string = "_"
while memory["version_scrape"]["last"] < VERSIONS:
for entry in NAMES:
print(entry)
""" print("DONE!")
s = scope(BRANCH)
if s.endswith("_"): # Branch into?
entries = BRANCH[s]
if entries is 0:
continue # Meh... we are at the end
else:
"""
#r = check()