Scraper

autoSchedule/__init__.py

@@ -0,0 +1,19 @@
+from time import time, sleep
+from threading import Thread
+
+from . import scrape
+
+from objects import glob
+
+def schedule_loop():
+	if glob.config["scrape"]["run_on_startup"]:
+		scrape.run()
+
+	while True:
+		_time = time() + glob.config["scrape"]["schedule_seconds"]
+		while _time - time() > 0:
+			sleep(1)
+		scrape.run()
+
+thread = Thread(target = schedule_loop)
+thread.start()

autoSchedule/scrape.py

@@ -0,0 +1,64 @@
+import urllib.request, json
+import time, calendar
+
+from objects import glob
+
+finished = True 
+
+def run():
+	global finished
+
+	if not finished:
+		print("[!] Scraper job is already running! ")
+		return
+
+	print("[!] Starting scraping job...")
+	sql = glob.new_sql()
+	cur = sql.cursor()
+	
+	cur.execute("SELECT max(file_version) f FROM updates")
+	target = cur.fetchone()["f"]
+
+	finished = False
+	failed_streak = 0
+	while not finished:
+		target += 1
+
+		attempts = 0
+		extra_sleep = 0
+		while attempts < glob.config["scrape"]["max_attempts"]:
+			try:
+				with urllib.request.urlopen("https://osu.ppy.sh/web/check-updates.php?action=path&stream=stable&target=%s" % target) as url:
+					data = json.loads(url.read().decode())[0]
+					if "url_patch" not in data.keys():
+						data["url_patch"] = None
+					cur.execute("INSERT INTO updates (file_version,filename,file_hash,filesize,timestamp,patch_id,url_full,url_patch) VALUES ('%s','%s','%s','%s',%s,'%s','%s','%s')" %
+						(
+							data["file_version"],
+							data["filename"],
+							data["file_hash"],
+							data["filesize"],
+							calendar.timegm(time.strptime(data["timestamp"], "%Y-%m-%d %H:%M:%S")),
+							data["patch_id"],
+							data["url_full"],
+							data["url_patch"],
+					))
+					sql.commit()
+					failed_streak = 0
+					print("[Scraper] Target: %s, Status: OK" % target)
+					break
+			except:
+				attempts += 1
+				failed_streak += 1
+				if glob.config["scrape"]["increase_delay_on_fail"]:
+					extra_sleep = attempts
+				print("[Scraper] Target: %s, status: FAILED, Attempt: %s" % (target, attempts))
+			
+			time.sleep(glob.config["scrape"]["delay"] + extra_sleep)
+			if failed_streak > glob.config["scrape"]["skips_until_finished"]:
+				finished = True
+				break
+	
+	cur.close()
+	sql.close()
+	print("[Scraper] Finished!")

config.json

@@ -3,23 +3,16 @@
         "host": "0.0.0.0",
         "port": 3003
     },
-    "sql": {
-        "host": "127.0.0.1",
-        "user": "root",
-        "passwd": "toor",
-        "db": "osu-wayback"
-    },
     "scrape": {
         "delay": 1,
-        "max_attempts": 3,
+		"max_attempts": 3,
-        "increase_delay_on_fail": true
+		"increase_delay_on_fail": true,
+		"skips_until_finished": 17,
+		"schedule_seconds": 68400,
+		"run_on_startup": true
 	},
 	"downloader": {
 		"download_folder": "/home/wayback/files"
     },
-    "zipper": {
-        "temp_folder": "/home/wayback/tmp",
-        "output_folder": "/home/wayback/archive"
-	},
 	"threads": 4
 }

main.py

@@ -76,4 +76,7 @@ if __name__ == "__main__":
 	print("To stop server press CTRL + C")
 	glob.app.listen(glob.config["server"]["port"], address=glob.config["server"]["host"])
 	tornado.log.enable_pretty_logging()
+
+	import autoSchedule
+
 	tornado.ioloop.IOLoop.instance().start()