From 0401acd31b0a58e1394538467fa5ce5e2950627e Mon Sep 17 00:00:00 2001 From: Sunpy Date: Mon, 11 Mar 2019 07:06:31 +0100 Subject: [PATCH] SQLite --- .gitignore | 5 +- .vscode/settings.json | 2 +- api/v1/getFile.py | 55 +++++++++++++++++-- config.json | 10 ++-- constants/__init__.py | 0 constants/argumentTypes.py | 4 ++ database_structs.sql | 17 ++++++ main.py | 36 ++++++++++++- objects/__init__.py | 0 objects/glob.py | 10 +++- web/__init__.py | 0 web/asyncTornado.py | 106 ++++++++++++++++++++++++++++++++++++- 12 files changed, 228 insertions(+), 17 deletions(-) create mode 100644 constants/__init__.py create mode 100644 constants/argumentTypes.py create mode 100644 database_structs.sql create mode 100644 objects/__init__.py create mode 100644 web/__init__.py diff --git a/.gitignore b/.gitignore index 7f0bfa3..727e551 100644 --- a/.gitignore +++ b/.gitignore @@ -98,4 +98,7 @@ ENV/ /site # mypy -.mypy_cache/ \ No newline at end of file +.mypy_cache/ + +# custom +wayback.db \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 1de9f05..2449d3f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,4 @@ { - "python.pythonPath": "/usr/bin/python3.6", + "python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", "python.linting.pylintEnabled": true } \ No newline at end of file diff --git a/api/v1/getFile.py b/api/v1/getFile.py index d6b4d56..0f04e31 100644 --- a/api/v1/getFile.py +++ b/api/v1/getFile.py @@ -1,12 +1,57 @@ +import json + import tornado.gen import tornado.web +from web import asyncTornado +from constants import argumentTypes + from objects import glob -allowed_args = ["file_hash", "file_version", "timestamp"] +ARGS = { + ("file_hash", "file_version", "timestamp"): argumentTypes.one_required +} -def handle(requestsManager.asyncRequestHandler): - return {} +SQL_STRUCT = { + "main": "SELECT * FROM updates WHERE %s LIMIT 1", + "file_hash": "%s = '%s'", + "file_version": "%s = %s", + "timestamp": "timestamp <= '%s' ORDER BY timestamp DESC" +} -def callback(method, data): - return None \ No newline at end of file +class handler(asyncTornado.asyncRequestHandler): + @tornado.web.asynchronous + @tornado.gen.engine + def asyncGet(self): + status_code = 400 + data = {} + try: + args_filter = asyncTornado.check_arguments(self.request.arguments, ARGS) + if False in args_filter: + raise Exception("Missing required arguments") + + method = args_filter[0] + method_value = self.request.arguments[method] + + cur = glob.sql.cursor() + + sql = SQL_STRUCT["main"] % SQL_STRUCT["method"] + if method == "timestamp": + sql = sql % method_value + else: + sql = sql % (method, method_value) + + cur.execute(sql) + data = cur.fetchone() + + status_code = 200 + except Exception as e: + status_code = 400 + data["status"] = status_code + data["message"] = e + finally: + cur.close() + + self.write( json.dumps(data) ) + self.set_header("Content-Type", "application/json") + self.set_status(status_code) diff --git a/config.json b/config.json index 6b9922f..4919365 100644 --- a/config.json +++ b/config.json @@ -1,9 +1,6 @@ { - "web": { - "debug": true, - "use_reloader": true, - "threaded": true, - "host": "127.0.0.1", + "server": { + "host": "0.0.0.0", "port": 3003 }, "sql": { @@ -23,5 +20,6 @@ "zipper": { "temp_folder": "/home/wayback/tmp", "output_folder": "/home/wayback/archive" - } + }, + "threads": 4 } \ No newline at end of file diff --git a/constants/__init__.py b/constants/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/constants/argumentTypes.py b/constants/argumentTypes.py new file mode 100644 index 0000000..b592046 --- /dev/null +++ b/constants/argumentTypes.py @@ -0,0 +1,4 @@ +optional = 0 +required = 1 +one_required = 2 +only_one = 3 diff --git a/database_structs.sql b/database_structs.sql new file mode 100644 index 0000000..b87eacb --- /dev/null +++ b/database_structs.sql @@ -0,0 +1,17 @@ +CREATE TABLE `updates` ( + `file_version` int(11) NOT NULL, + `filename` varchar(32) NOT NULL, + `file_hash` varchar(32) NOT NULL, + `filesize` int(11) NOT NULL, + `timestamp` int(11) NOT NULL, + `patch_id` int(11) DEFAULT NULL, + `url_full` varchar(128) NOT NULL, + `url_patch` varchar(128) DEFAULT NULL, + PRIMARY KEY (`file_version`) +); + +CREATE TABLE `osu_builds` ( + `id` int(11) NOT NULL, + `version` varchar(128) NOT NULL, + PRIMARY KEY (`id`) +); \ No newline at end of file diff --git a/main.py b/main.py index f9a31bd..2c09d42 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,6 @@ -from os import listdir +from os import listdir, path + +import sqlite3 import tornado.gen import tornado.httpserver @@ -6,6 +8,9 @@ import tornado.ioloop import tornado.web import tornado.netutil + +from objects import glob + def make_app(): """ Make tornado application instance @@ -35,7 +40,7 @@ def make_app(): api = api.rstrip(".py") routes.append( ( - r"/%s/%s" % (dir, api), __import__("%s.%s" % (dir.replace("/", "."), api), fromlist=[""]).handle + r"/%s/%s" % (dir, api), __import__("%s.%s" % (dir.replace("/", "."), api), fromlist=[""]) )) else: routes += map_routes("%s/%s" % (dir, api)) @@ -43,3 +48,30 @@ def make_app(): routes = map_routes("api") return tornado.web.Application(routes) + +def build_database(): + f = open("wayback.db", "w") + f.close() + + glob.sql = sqlite3.connect("wayback.db") + cur = glob.sql.cursor() + + with open("database_structs.sql", "r") as f: + cur.executescript( f.read() ) + + cur.close() + print("[!] New sqlite database created") + +if __name__ == "__main__": + glob.app = make_app() + + if not path.isfile("wayback.db"): + build_database() + + if glob.sql == None: + glob.sql = sqlite3.connect("wayback.db") + + print("Serving at %s:%s" % (glob.config["server"]["host"], glob.config["server"]["port"])) + print("To stop server press CTRL + C") + glob.app.listen(glob.config["server"]["port"], address=glob.config["server"]["host"]) + tornado.ioloop.IOLoop.instance().start() \ No newline at end of file diff --git a/objects/__init__.py b/objects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/objects/glob.py b/objects/glob.py index 1d87e23..26391cf 100644 --- a/objects/glob.py +++ b/objects/glob.py @@ -1 +1,9 @@ -sql = None \ No newline at end of file +import json +from multiprocessing.pool import ThreadPool + +with open("config.json", "r") as f: + config = json.load(f) + +app = None +sql = None +pool = ThreadPool(config["threads"]) diff --git a/web/__init__.py b/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/web/asyncTornado.py b/web/asyncTornado.py index 1f02d36..59cda8c 100644 --- a/web/asyncTornado.py +++ b/web/asyncTornado.py @@ -1,6 +1,110 @@ import tornado import tornado.web import tornado.gen +from tornado.ioloop import IOLoop +from constants import argumentTypes +from objects import glob + +# Copied from https://zxq.co/ripple/ripple-python-common/src/branch/master/web/requestsManager.py class asyncRequestHandler(tornado.web.RequestHandler): - \ No newline at end of file + """ + Tornado asynchronous request handler + create a class that extends this one (requestHelper.asyncRequestHandler) + use asyncGet() and asyncPost() instead of get() and post(). + Done. I'm not kidding. + """ + @tornado.web.asynchronous + @tornado.gen.engine + def get(self, *args, **kwargs): + try: + yield tornado.gen.Task(runBackground, (self.asyncGet, tuple(args), dict(kwargs))) + finally: + if not self._finished: + self.finish() + + @tornado.web.asynchronous + @tornado.gen.engine + def post(self, *args, **kwargs): + try: + yield tornado.gen.Task(runBackground, (self.asyncPost, tuple(args), dict(kwargs))) + finally: + if not self._finished: + self.finish() + + def asyncGet(self, *args, **kwargs): + self.send_error(405) + + def asyncPost(self, *args, **kwargs): + self.send_error(405) + + def getRequestIP(self): + """ + Return CF-Connecting-IP (request IP when under cloudflare, you have to configure nginx to enable that) + If that fails, return X-Forwarded-For (request IP when not under Cloudflare) + if everything else fails, return remote IP + + :return: Client IP address + """ + if "CF-Connecting-IP" in self.request.headers: + return self.request.headers.get("CF-Connecting-IP") + elif "X-Forwarded-For" in self.request.headers: + return self.request.headers.get("X-Forwarded-For") + else: + return self.request.remote_ip + + +def runBackground(data, callback): + """ + Run a function in the background. + Used to handle multiple requests at the same time + + :param data: (func, args, kwargs) + :param callback: function to call when `func` (data[0]) returns + :return: + """ + func, args, kwargs = data + def _callback(result): + IOLoop.instance().add_callback(lambda: callback(result)) + glob.pool.apply_async(func, args, kwargs, _callback) + +def check_arguments(arguments, arguments_filter): + filter_pass = [] + for k, v in arguments_filter.items(): + if v == argumentTypes.optional: + filter_pass.append( arg_filter_and(arguments, k) ) + elif v == argumentTypes.required: + filter_pass.append( arg_filter_require_all(arguments, k) ) + elif v == argumentTypes.one_required: + filter_pass.append( arg_filter_first(arguments, k, False) ) + elif v == argumentTypes.only_one: + filter_pass.append( arg_filter_only_one(arguments, k) ) + return filter_pass + +def arg_filter_and(arguments, filter, can_false = False): + arg_filter = [] + for i in filter: + if i in filter: + arg_filter.append(i) + if can_false: + return arg_filter if len(arg_filter) else False + return arg_filter + +def arg_filter_require_all(arguments, required): + for i in required: + if i not in arguments: + return False + return required + +def arg_filter_only_one(arguments, required): + arg_filter = [] + for i in required: + if i in arguments: + arg_filter.append(i) + return True if len(arg_filter) == 1 else False + +def arg_filter_first(arguments, filter, optional = True): + for i in filter: + if i in filter: + return i + return optional