Merge branch 'master' of https://github.com/osufx/osu-wayback

config.json

@@ -16,5 +16,8 @@
         "delay": 1,
         "max_attempts": 3,
         "increase_delay_on_fail": true
+	},
+	"downloader": {
+		"download_folder": "/home/wayback/files"
 	}
 }

downloader.py

@@ -0,0 +1,47 @@
+#Used to download all the files into local folders after scrape data has been saved
+
+import urllib.request, json
+import MySQLdb
+import MySQLdb.cursors
+import os
+import atexit
+
+with open("config.json", "r") as f:
+    config = json.load(f)
+
+with open("memory.json", "r") as f:
+    memory = json.load(f)
+
+sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
+
+cur = sql.cursor()
+
+def on_close():
+    with open("memory.json", "w") as f:
+        json.dump(memory, f)
+    print("Closing...")
+
+atexit.register(on_close)
+
+cur.execute("SELECT file_version,filename,file_hash,url_full FROM updates")
+data = cur.fetchall()
+
+# Remove already downloaded files (checked from memory.json)
+data = data[memory["downloader"]["last"]:]
+
+for row in data:
+	try:
+		print("Downloading {} with id {}".format(row["filename"], row["file_version"]))
+		urllib.request.urlretrieve(
+				row["url_full"],
+				os.path.join(
+					config["downloader"]["download_folder"],
+					row["filename"],
+					"f_" + row["file_hash"]
+				)
+			)
+		print("Done.")
+	except Exception as e:
+		memory["downloader"]["failed"].append(row["file_version"])
+		print("Error downloading file {}: {}".format(row["file_version"], e))
+	memory["downloader"]["last"] += 1

handlers/update.py

@@ -0,0 +1,2 @@
+def handle(request):
+	return "Unimplemented"

memory.json

@@ -1 +1,10 @@
-{"scrape": {"last": 0, "failed": []}}
+{
+	"scrape": {
+		"last": 0,
+		"failed": []
+	},
+	"downloader": {
+		"last": 0,
+		"failed": []
+	}
+}

templates/api.html

@@ -0,0 +1 @@
+Unimplemented

web.py

@@ -2,6 +2,7 @@ import json
 import MySQLdb
 import MySQLdb.cursors
 from flask import Flask, make_response, request, render_template, jsonify
+from handlers import update
 
 app = Flask(__name__)
 
@@ -21,5 +22,13 @@ def download_index():
 def api_index():
 	return render_template("api.html")
 
+@app.route("/api/getUpdate", methods=["GET", "POST"])
+def api_update():
+	return update.handle(request)
+
+@app.route("/api")
+def api_index():
+    return render_template("api.html")
+
 if __name__ == "__main__":
 	app.run(**config["web"])