Added base for zipper
This commit is contained in:
parent
a4f91d5acf
commit
76e86ab94a
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
|
@ -1,3 +1,3 @@
|
||||||
{
|
{
|
||||||
"python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe"
|
"python.pythonPath": "/usr/bin/python3.6"
|
||||||
}
|
}
|
|
@ -19,5 +19,9 @@
|
||||||
},
|
},
|
||||||
"downloader": {
|
"downloader": {
|
||||||
"download_folder": "/home/wayback/files"
|
"download_folder": "/home/wayback/files"
|
||||||
|
},
|
||||||
|
"zipper": {
|
||||||
|
"temp_folder": "/home/wayback/tmp",
|
||||||
|
"output_folder": "/home/wayback/archive"
|
||||||
}
|
}
|
||||||
}
|
}
|
17
handlers/getupdate.py
Normal file
17
handlers/getupdate.py
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
allowed_args = ["file_hash", "file_version", "timestamp"]
|
||||||
|
|
||||||
|
def handle(request):
|
||||||
|
if len([x for x in request.args if x in allowed_args]) == 0:
|
||||||
|
return {"error": "Missing valid args"}
|
||||||
|
|
||||||
|
for i in range(len(allowed_args)): # Gets the first valid argument and sets it as the method handler
|
||||||
|
method = request.args.get(allowed_args[i])
|
||||||
|
method_name = allowed_args[i]
|
||||||
|
if method is not None:
|
||||||
|
break
|
||||||
|
|
||||||
|
output = {
|
||||||
|
"search_method": method_name
|
||||||
|
}
|
||||||
|
output["endpoint"] = request.endpoint
|
||||||
|
return output
|
|
@ -6,5 +6,8 @@
|
||||||
"downloader": {
|
"downloader": {
|
||||||
"last": 0,
|
"last": 0,
|
||||||
"failed": []
|
"failed": []
|
||||||
|
},
|
||||||
|
"zipper": {
|
||||||
|
"last": 0
|
||||||
}
|
}
|
||||||
}
|
}
|
50
zipper.py
Normal file
50
zipper.py
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
#Used to zip all the files into new local folders after downloader is done
|
||||||
|
|
||||||
|
import urllib.request, json
|
||||||
|
import MySQLdb
|
||||||
|
import MySQLdb.cursors
|
||||||
|
import os
|
||||||
|
import atexit
|
||||||
|
|
||||||
|
with open("config.json", "r") as f:
|
||||||
|
config = json.load(f)
|
||||||
|
|
||||||
|
with open("memory.json", "r") as f:
|
||||||
|
memory = json.load(f)
|
||||||
|
|
||||||
|
sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
|
||||||
|
|
||||||
|
cur = sql.cursor()
|
||||||
|
|
||||||
|
def on_close():
|
||||||
|
with open("memory.json", "w") as f:
|
||||||
|
json.dump(memory, f)
|
||||||
|
print("Closing...")
|
||||||
|
|
||||||
|
atexit.register(on_close)
|
||||||
|
|
||||||
|
cur.execute("SELECT file_version,filename,file_hash,url_full FROM updates")
|
||||||
|
data = cur.fetchall()
|
||||||
|
|
||||||
|
# Remove already downloaded files (checked from memory.json)
|
||||||
|
data = data[memory["zipper"]["last"]:]
|
||||||
|
|
||||||
|
# Unfinished - replace with zipper code
|
||||||
|
"""
|
||||||
|
for row in data:
|
||||||
|
try:
|
||||||
|
print("Downloading {} with id {}".format(row["filename"], row["file_version"]))
|
||||||
|
urllib.request.urlretrieve(
|
||||||
|
row["url_full"],
|
||||||
|
os.path.join(
|
||||||
|
config["downloader"]["download_folder"],
|
||||||
|
row["filename"],
|
||||||
|
"f_" + row["file_hash"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
print("Done.")
|
||||||
|
except Exception as e:
|
||||||
|
memory["downloader"]["failed"].append(row["file_version"])
|
||||||
|
print("Error downloading file {}: {}".format(row["file_version"], e))
|
||||||
|
memory["downloader"]["last"] += 1
|
||||||
|
"""
|
Loading…
Reference in New Issue
Block a user