Added base for zipper

This commit is contained in:
Emily 2018-05-24 12:22:16 +02:00
parent a4f91d5acf
commit 76e86ab94a
5 changed files with 76 additions and 2 deletions

View File

@ -1,3 +1,3 @@
{ {
"python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe" "python.pythonPath": "/usr/bin/python3.6"
} }

View File

@ -19,5 +19,9 @@
}, },
"downloader": { "downloader": {
"download_folder": "/home/wayback/files" "download_folder": "/home/wayback/files"
} },
"zipper": {
"temp_folder": "/home/wayback/tmp",
"output_folder": "/home/wayback/archive"
}
} }

17
handlers/getupdate.py Normal file
View File

@ -0,0 +1,17 @@
allowed_args = ["file_hash", "file_version", "timestamp"]
def handle(request):
if len([x for x in request.args if x in allowed_args]) == 0:
return {"error": "Missing valid args"}
for i in range(len(allowed_args)): # Gets the first valid argument and sets it as the method handler
method = request.args.get(allowed_args[i])
method_name = allowed_args[i]
if method is not None:
break
output = {
"search_method": method_name
}
output["endpoint"] = request.endpoint
return output

View File

@ -6,5 +6,8 @@
"downloader": { "downloader": {
"last": 0, "last": 0,
"failed": [] "failed": []
},
"zipper": {
"last": 0
} }
} }

50
zipper.py Normal file
View File

@ -0,0 +1,50 @@
#Used to zip all the files into new local folders after downloader is done
import urllib.request, json
import MySQLdb
import MySQLdb.cursors
import os
import atexit
with open("config.json", "r") as f:
config = json.load(f)
with open("memory.json", "r") as f:
memory = json.load(f)
sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
cur = sql.cursor()
def on_close():
with open("memory.json", "w") as f:
json.dump(memory, f)
print("Closing...")
atexit.register(on_close)
cur.execute("SELECT file_version,filename,file_hash,url_full FROM updates")
data = cur.fetchall()
# Remove already downloaded files (checked from memory.json)
data = data[memory["zipper"]["last"]:]
# Unfinished - replace with zipper code
"""
for row in data:
try:
print("Downloading {} with id {}".format(row["filename"], row["file_version"]))
urllib.request.urlretrieve(
row["url_full"],
os.path.join(
config["downloader"]["download_folder"],
row["filename"],
"f_" + row["file_hash"]
)
)
print("Done.")
except Exception as e:
memory["downloader"]["failed"].append(row["file_version"])
print("Error downloading file {}: {}".format(row["file_version"], e))
memory["downloader"]["last"] += 1
"""