Compare commits

...

4 Commits

Author SHA1 Message Date
e3eaba33ed sql row factory 2019-03-11 07:51:05 +01:00
fa152f4d46 Error handling 2019-03-11 07:28:53 +01:00
416d7fdd6d Fixed typo 2019-03-11 07:27:54 +01:00
0401acd31b SQLite 2019-03-11 07:06:31 +01:00
18 changed files with 262 additions and 55 deletions

3
.gitignore vendored
View File

@ -99,3 +99,6 @@ ENV/
# mypy
.mypy_cache/
# custom
wayback.db

View File

@ -1,4 +1,4 @@
{
"python.pythonPath": "/usr/bin/python3.6",
"python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe",
"python.linting.pylintEnabled": true
}

View File

@ -1,12 +1,60 @@
import json
import tornado.gen
import tornado.web
from web import asyncTornado
from constants import argumentTypes
from objects import glob
allowed_args = ["file_hash", "file_version", "timestamp"]
ARGS = {
("file_hash", "file_version", "timestamp"): argumentTypes.one_required
}
def handle(requestsManager.asyncRequestHandler):
return {}
SQL_STRUCT = {
"main": "SELECT * FROM updates WHERE %s LIMIT 1",
"file_hash": "%s = '%s'",
"file_version": "%s = %s",
"timestamp": "timestamp <= '%s' ORDER BY timestamp DESC"
}
def callback(method, data):
return None
class handler(asyncTornado.asyncRequestHandler):
@tornado.web.asynchronous
@tornado.gen.engine
def asyncGet(self):
status_code = 400
data = {}
try:
args_filter = asyncTornado.check_arguments(self.request.arguments, ARGS)
if False in args_filter:
raise Exception("Missing required arguments")
method = args_filter[0]
method_value = self.get_argument(method)
conn = glob.new_sql()
cur = conn.cursor()
sql = SQL_STRUCT["main"] % SQL_STRUCT[method]
if method == "timestamp":
sql = sql % method_value
else:
sql = sql % (method, method_value)
cur.execute(sql)
data = cur.fetchone()
cur.close()
conn.close()
status_code = 200
except Exception as e:
status_code = 400
data["status"] = status_code
data["message"] = str(e)
print("Err: %s" % e)
finally:
self.write( json.dumps(data) )
self.set_header("Content-Type", "application/json")
self.set_status(status_code)

View File

@ -1,9 +1,6 @@
{
"web": {
"debug": true,
"use_reloader": true,
"threaded": true,
"host": "127.0.0.1",
"server": {
"host": "0.0.0.0",
"port": 3003
},
"sql": {
@ -23,5 +20,6 @@
"zipper": {
"temp_folder": "/home/wayback/tmp",
"output_folder": "/home/wayback/archive"
}
},
"threads": 4
}

0
constants/__init__.py Normal file
View File

View File

@ -0,0 +1,4 @@
optional = 0
required = 1
one_required = 2
only_one = 3

17
database_structs.sql Normal file
View File

@ -0,0 +1,17 @@
CREATE TABLE `updates` (
`file_version` int(11) NOT NULL,
`filename` varchar(32) NOT NULL,
`file_hash` varchar(32) NOT NULL,
`filesize` int(11) NOT NULL,
`timestamp` int(11) NOT NULL,
`patch_id` int(11) DEFAULT NULL,
`url_full` varchar(128) NOT NULL,
`url_patch` varchar(128) DEFAULT NULL,
PRIMARY KEY (`file_version`)
);
CREATE TABLE `osu_builds` (
`id` int(11) NOT NULL,
`version` varchar(128) NOT NULL,
PRIMARY KEY (`id`)
);

View File

@ -1,38 +0,0 @@
import json
#import MySQLdb
#import MySQLdb.cursors
from flask import Flask, make_response, request, render_template, jsonify
from objects import glob
import api
app = Flask(__name__)
with open("config.json", "r") as f:
config = json.load(f)
# Setup sql
#glob.sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
@app.route("/")
@app.route("/home")
def home_index():
return render_template("index.html")
@app.route("/download")
def download_index():
return render_template("download.html")
@app.route("/api")
def api_index():
return render_template("api.html")
@app.route("/api/<ver>/<func>")
def api_call(ver, func):
if ver not in api.versions or func not in api.versions[ver].handlers:
return jsonify(api.missing.handle(request))
data = api.versions[ver].handlers[func].handle(request)
return jsonify(data)
if __name__ == "__main__":
app.run(**config["web"])

0
helpers/__init__.py Normal file
View File

View File

@ -0,0 +1,5 @@
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d

19
import_database.py Normal file
View File

@ -0,0 +1,19 @@
if __name__ != "__main__":
print("This is not a module")
exit()
import sys
import sqlite3
if len(sys.argv) < 2:
print("Missing database import file")
exit()
sql = sqlite3.connect("wayback.db")
cur = sql.cursor()
with open(sys.argv[1], "r") as f:
cur.executescript( f.read() )
cur.close()
sql.close()

36
main.py
View File

@ -1,4 +1,6 @@
from os import listdir
from os import listdir, path
import sqlite3
import tornado.gen
import tornado.httpserver
@ -6,6 +8,9 @@ import tornado.ioloop
import tornado.web
import tornado.netutil
from objects import glob
def make_app():
"""
Make tornado application instance
@ -35,7 +40,7 @@ def make_app():
api = api.rstrip(".py")
routes.append(
(
r"/%s/%s" % (dir, api), __import__("%s.%s" % (dir.replace("/", "."), api), fromlist=[""]).handle
r"/%s/%s" % (dir, api), __import__("%s.%s" % (dir.replace("/", "."), api), fromlist=[""]).handler
))
else:
routes += map_routes("%s/%s" % (dir, api))
@ -43,3 +48,30 @@ def make_app():
routes = map_routes("api")
return tornado.web.Application(routes)
def build_database():
f = open("wayback.db", "w")
f.close()
glob.sql = glob.new_sql()
cur = glob.sql.cursor()
with open("database_structs.sql", "r") as f:
cur.executescript( f.read() )
cur.close()
print("[!] New sqlite database created")
if __name__ == "__main__":
glob.app = make_app()
if not path.isfile("wayback.db"):
build_database()
if glob.sql == None:
glob.sql = glob.new_sql()
print("Serving at %s:%s" % (glob.config["server"]["host"], glob.config["server"]["port"]))
print("To stop server press CTRL + C")
glob.app.listen(glob.config["server"]["port"], address=glob.config["server"]["host"])
tornado.ioloop.IOLoop.instance().start()

0
objects/__init__.py Normal file
View File

View File

@ -1 +1,16 @@
import json
import sqlite3
from multiprocessing.pool import ThreadPool
from helpers import databaseHelper
with open("config.json", "r") as f:
config = json.load(f)
app = None
sql = None
pool = ThreadPool(config["threads"])
def new_sql():
db = sqlite3.connect("wayback.db")
db.row_factory = databaseHelper.dict_factory
return db

0
web/__init__.py Normal file
View File

View File

@ -1,6 +1,110 @@
import tornado
import tornado.web
import tornado.gen
from tornado.ioloop import IOLoop
from constants import argumentTypes
from objects import glob
# Copied from https://zxq.co/ripple/ripple-python-common/src/branch/master/web/requestsManager.py
class asyncRequestHandler(tornado.web.RequestHandler):
"""
Tornado asynchronous request handler
create a class that extends this one (requestHelper.asyncRequestHandler)
use asyncGet() and asyncPost() instead of get() and post().
Done. I'm not kidding.
"""
@tornado.web.asynchronous
@tornado.gen.engine
def get(self, *args, **kwargs):
try:
yield tornado.gen.Task(runBackground, (self.asyncGet, tuple(args), dict(kwargs)))
finally:
if not self._finished:
self.finish()
@tornado.web.asynchronous
@tornado.gen.engine
def post(self, *args, **kwargs):
try:
yield tornado.gen.Task(runBackground, (self.asyncPost, tuple(args), dict(kwargs)))
finally:
if not self._finished:
self.finish()
def asyncGet(self, *args, **kwargs):
self.send_error(405)
def asyncPost(self, *args, **kwargs):
self.send_error(405)
def getRequestIP(self):
"""
Return CF-Connecting-IP (request IP when under cloudflare, you have to configure nginx to enable that)
If that fails, return X-Forwarded-For (request IP when not under Cloudflare)
if everything else fails, return remote IP
:return: Client IP address
"""
if "CF-Connecting-IP" in self.request.headers:
return self.request.headers.get("CF-Connecting-IP")
elif "X-Forwarded-For" in self.request.headers:
return self.request.headers.get("X-Forwarded-For")
else:
return self.request.remote_ip
def runBackground(data, callback):
"""
Run a function in the background.
Used to handle multiple requests at the same time
:param data: (func, args, kwargs)
:param callback: function to call when `func` (data[0]) returns
:return:
"""
func, args, kwargs = data
def _callback(result):
IOLoop.instance().add_callback(lambda: callback(result))
glob.pool.apply_async(func, args, kwargs, _callback)
def check_arguments(arguments, arguments_filter):
filter_pass = []
for k, v in arguments_filter.items():
if v == argumentTypes.optional:
filter_pass.append( arg_filter_and(arguments, k) )
elif v == argumentTypes.required:
filter_pass.append( arg_filter_require_all(arguments, k) )
elif v == argumentTypes.one_required:
filter_pass.append( arg_filter_first(arguments, k, False) )
elif v == argumentTypes.only_one:
filter_pass.append( arg_filter_only_one(arguments, k) )
return filter_pass
def arg_filter_and(arguments, filter, can_false = False):
arg_filter = []
for i in filter:
if i in arguments:
arg_filter.append(i)
if can_false:
return arg_filter if len(arg_filter) else False
return arg_filter
def arg_filter_require_all(arguments, required):
for i in required:
if i not in arguments:
return False
return required
def arg_filter_only_one(arguments, required):
arg_filter = []
for i in required:
if i in arguments:
arg_filter.append(i)
return True if len(arg_filter) == 1 else False
def arg_filter_first(arguments, filter, optional = True):
for i in filter:
if i in arguments:
return i
return optional