Added base for zipper

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-	"python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe"
+	"python.pythonPath": "/usr/bin/python3.6"
 }

config.json

@@ -19,5 +19,9 @@
 	},
 	"downloader": {
 		"download_folder": "/home/wayback/files"
-	}
+    },
+    "zipper": {
+        "temp_folder": "/home/wayback/tmp",
+        "output_folder": "/home/wayback/archive"
+    }
 }

handlers/getupdate.py

@@ -0,0 +1,17 @@
+allowed_args = ["file_hash", "file_version", "timestamp"]
+
+def handle(request):
+    if len([x for x in request.args if x in allowed_args]) == 0:
+        return {"error": "Missing valid args"}
+
+    for i in range(len(allowed_args)): # Gets the first valid argument and sets it as the method handler
+        method = request.args.get(allowed_args[i])
+        method_name = allowed_args[i]
+        if method is not None:
+            break
+
+    output = {
+        "search_method": method_name
+    }
+    output["endpoint"] = request.endpoint
+    return output

memory.json

@@ -6,5 +6,8 @@
 	"downloader": {
 		"last": 0,
 		"failed": []
+	},
+	"zipper": {
+		"last": 0
 	}
 }

zipper.py

@@ -0,0 +1,50 @@
+#Used to zip all the files into new local folders after downloader is done
+
+import urllib.request, json
+import MySQLdb
+import MySQLdb.cursors
+import os
+import atexit
+
+with open("config.json", "r") as f:
+    config = json.load(f)
+
+with open("memory.json", "r") as f:
+    memory = json.load(f)
+
+sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
+
+cur = sql.cursor()
+
+def on_close():
+    with open("memory.json", "w") as f:
+        json.dump(memory, f)
+    print("Closing...")
+
+atexit.register(on_close)
+
+cur.execute("SELECT file_version,filename,file_hash,url_full FROM updates")
+data = cur.fetchall()
+
+# Remove already downloaded files (checked from memory.json)
+data = data[memory["zipper"]["last"]:]
+
+# Unfinished - replace with zipper code
+"""
+for row in data:
+	try:
+		print("Downloading {} with id {}".format(row["filename"], row["file_version"]))
+		urllib.request.urlretrieve(
+				row["url_full"],
+				os.path.join(
+					config["downloader"]["download_folder"],
+					row["filename"],
+					"f_" + row["file_hash"]
+				)
+			)
+		print("Done.")
+	except Exception as e:
+		memory["downloader"]["failed"].append(row["file_version"])
+		print("Error downloading file {}: {}".format(row["file_version"], e))
+	memory["downloader"]["last"] += 1
+"""