diff --git a/.vscode/settings.json b/.vscode/settings.json index da7c64d..59fa702 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { - "python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe" + "python.pythonPath": "/usr/bin/python3.6" } \ No newline at end of file diff --git a/config.json b/config.json index 7e8f8e4..6b9922f 100644 --- a/config.json +++ b/config.json @@ -19,5 +19,9 @@ }, "downloader": { "download_folder": "/home/wayback/files" - } + }, + "zipper": { + "temp_folder": "/home/wayback/tmp", + "output_folder": "/home/wayback/archive" + } } \ No newline at end of file diff --git a/handlers/getupdate.py b/handlers/getupdate.py new file mode 100644 index 0000000..330d2af --- /dev/null +++ b/handlers/getupdate.py @@ -0,0 +1,17 @@ +allowed_args = ["file_hash", "file_version", "timestamp"] + +def handle(request): + if len([x for x in request.args if x in allowed_args]) == 0: + return {"error": "Missing valid args"} + + for i in range(len(allowed_args)): # Gets the first valid argument and sets it as the method handler + method = request.args.get(allowed_args[i]) + method_name = allowed_args[i] + if method is not None: + break + + output = { + "search_method": method_name + } + output["endpoint"] = request.endpoint + return output \ No newline at end of file diff --git a/memory.json b/memory.json index 7500ded..d3d8fb6 100644 --- a/memory.json +++ b/memory.json @@ -6,5 +6,8 @@ "downloader": { "last": 0, "failed": [] + }, + "zipper": { + "last": 0 } } \ No newline at end of file diff --git a/zipper.py b/zipper.py new file mode 100644 index 0000000..b91341b --- /dev/null +++ b/zipper.py @@ -0,0 +1,50 @@ +#Used to zip all the files into new local folders after downloader is done + +import urllib.request, json +import MySQLdb +import MySQLdb.cursors +import os +import atexit + +with open("config.json", "r") as f: + config = json.load(f) + +with open("memory.json", "r") as f: + memory = json.load(f) + +sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor) + +cur = sql.cursor() + +def on_close(): + with open("memory.json", "w") as f: + json.dump(memory, f) + print("Closing...") + +atexit.register(on_close) + +cur.execute("SELECT file_version,filename,file_hash,url_full FROM updates") +data = cur.fetchall() + +# Remove already downloaded files (checked from memory.json) +data = data[memory["zipper"]["last"]:] + +# Unfinished - replace with zipper code +""" +for row in data: + try: + print("Downloading {} with id {}".format(row["filename"], row["file_version"])) + urllib.request.urlretrieve( + row["url_full"], + os.path.join( + config["downloader"]["download_folder"], + row["filename"], + "f_" + row["file_hash"] + ) + ) + print("Done.") + except Exception as e: + memory["downloader"]["failed"].append(row["file_version"]) + print("Error downloading file {}: {}".format(row["file_version"], e)) + memory["downloader"]["last"] += 1 +""" \ No newline at end of file