sql row factory

.gitignore

@@ -98,4 +98,7 @@ ENV/
 /site
 
 # mypy
-.mypy_cache/
+.mypy_cache/
+
+# custom
+wayback.db

.vscode/settings.json

@@ -1,4 +1,4 @@
 {
-	"python.pythonPath": "/usr/bin/python3.6",
+	"python.pythonPath": "C:\\Users\\Emily\\AppData\\Local\\Programs\\Python\\Python36\\python.exe",
 	"python.linting.pylintEnabled": true
 }

api/v1/getFile.py

@@ -1,12 +1,60 @@
+import json
+
 import tornado.gen
 import tornado.web
 
+from web import asyncTornado
+from constants import argumentTypes
+
 from objects import glob
 
-allowed_args = ["file_hash", "file_version", "timestamp"]
+ARGS = {
+	("file_hash", "file_version", "timestamp"): argumentTypes.one_required
+}
 
-def handle(requestsManager.asyncRequestHandler):
+SQL_STRUCT = {
-    return {}
+	"main": "SELECT * FROM updates WHERE %s LIMIT 1",
+	"file_hash": "%s = '%s'",
+	"file_version": "%s = %s",
+	"timestamp": "timestamp <= '%s' ORDER BY timestamp DESC"
+}
 
-def callback(method, data):
+class handler(asyncTornado.asyncRequestHandler):
-    return None
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def asyncGet(self):
+		status_code = 400
+		data = {}
+		try:
+			args_filter = asyncTornado.check_arguments(self.request.arguments, ARGS)
+			if False in args_filter:
+				raise Exception("Missing required arguments")
+
+			method = args_filter[0]
+			method_value = self.get_argument(method)
+
+			conn = glob.new_sql()
+			cur = conn.cursor()
+
+			sql = SQL_STRUCT["main"] % SQL_STRUCT[method]
+			if method == "timestamp":
+				sql = sql % method_value
+			else:
+				sql = sql % (method, method_value)
+
+			cur.execute(sql)
+			data = cur.fetchone()
+
+			cur.close()
+			conn.close()
+
+			status_code = 200
+		except Exception as e:
+			status_code = 400
+			data["status"] = status_code
+			data["message"] = str(e)
+			print("Err: %s" % e)
+		finally:
+			self.write( json.dumps(data) )
+			self.set_header("Content-Type", "application/json")
+			self.set_status(status_code)

config.json

@@ -1,9 +1,6 @@
 {
-    "web": {
+    "server": {
-        "debug": true,
+        "host": "0.0.0.0",
-        "use_reloader": true,
-        "threaded": true,
-        "host": "127.0.0.1",
         "port": 3003
     },
     "sql": {
@@ -23,5 +20,6 @@
     "zipper": {
         "temp_folder": "/home/wayback/tmp",
         "output_folder": "/home/wayback/archive"
-    }
+	},
+	"threads": 4
 }

constants/argumentTypes.py

@@ -0,0 +1,4 @@
+optional		= 0
+required		= 1
+one_required	= 2
+only_one		= 3

database_structs.sql

@@ -0,0 +1,17 @@
+CREATE TABLE `updates` (
+  `file_version` int(11) NOT NULL,
+  `filename` varchar(32) NOT NULL,
+  `file_hash` varchar(32) NOT NULL,
+  `filesize` int(11) NOT NULL,
+  `timestamp` int(11) NOT NULL,
+  `patch_id` int(11) DEFAULT NULL,
+  `url_full` varchar(128) NOT NULL,
+  `url_patch` varchar(128) DEFAULT NULL,
+  PRIMARY KEY (`file_version`)
+);
+
+CREATE TABLE `osu_builds` (
+  `id` int(11) NOT NULL,
+  `version` varchar(128) NOT NULL,
+  PRIMARY KEY (`id`)
+);

flask_main.py

@@ -1,38 +0,0 @@
-import json
-#import MySQLdb
-#import MySQLdb.cursors
-from flask import Flask, make_response, request, render_template, jsonify
-from objects import glob
-
-import api
-
-app = Flask(__name__)
-
-with open("config.json", "r") as f:
-	config = json.load(f)
-
-# Setup sql
-#glob.sql = MySQLdb.connect(**config["sql"], cursorclass = MySQLdb.cursors.DictCursor)
-
-@app.route("/")
-@app.route("/home")
-def home_index():
-	return render_template("index.html")
-
-@app.route("/download")
-def download_index():
-	return render_template("download.html")
-
-@app.route("/api")
-def api_index():
-	return render_template("api.html")
-
-@app.route("/api/<ver>/<func>")
-def api_call(ver, func):
-	if ver not in api.versions or func not in api.versions[ver].handlers:
-		return jsonify(api.missing.handle(request))
-	data = api.versions[ver].handlers[func].handle(request)
-	return jsonify(data)
-
-if __name__ == "__main__":
-	app.run(**config["web"])

helpers/databaseHelper.py

@@ -0,0 +1,5 @@
+def dict_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+    return d

import_database.py

@@ -0,0 +1,19 @@
+if __name__ != "__main__":
+	print("This is not a module")
+	exit()
+
+import sys
+import sqlite3
+
+if len(sys.argv) < 2:
+	print("Missing database import file")
+	exit()
+
+sql = sqlite3.connect("wayback.db")
+cur = sql.cursor()
+
+with open(sys.argv[1], "r") as f:
+	cur.executescript( f.read() )
+
+cur.close()
+sql.close()

main.py

@@ -1,4 +1,6 @@
-from os import listdir
+from os import listdir, path
+
+import sqlite3
 
 import tornado.gen
 import tornado.httpserver
@@ -6,6 +8,9 @@ import tornado.ioloop
 import tornado.web
 import tornado.netutil
 
+
+from objects import glob
+
 def make_app():
     """
     Make tornado application instance
@@ -35,7 +40,7 @@ def make_app():
                 api = api.rstrip(".py")
                 routes.append(
                     (
-                        r"/%s/%s" % (dir, api), __import__("%s.%s" % (dir.replace("/", "."), api), fromlist=[""]).handle
+                        r"/%s/%s" % (dir, api), __import__("%s.%s" % (dir.replace("/", "."), api), fromlist=[""]).handler
                     ))
             else:
                 routes += map_routes("%s/%s" % (dir, api))
@@ -43,3 +48,30 @@ def make_app():
 
     routes = map_routes("api")
     return tornado.web.Application(routes)
+
+def build_database():
+	f = open("wayback.db", "w")
+	f.close()
+
+	glob.sql = glob.new_sql()
+	cur = glob.sql.cursor()
+
+	with open("database_structs.sql", "r") as f:
+		cur.executescript( f.read() )
+
+	cur.close()
+	print("[!] New sqlite database created")
+
+if __name__ == "__main__":
+	glob.app = make_app()
+
+	if not path.isfile("wayback.db"):
+		build_database()
+
+	if glob.sql == None:
+		glob.sql = glob.new_sql()
+
+	print("Serving at %s:%s" % (glob.config["server"]["host"], glob.config["server"]["port"]))
+	print("To stop server press CTRL + C")
+	glob.app.listen(glob.config["server"]["port"], address=glob.config["server"]["host"])
+	tornado.ioloop.IOLoop.instance().start()

objects/glob.py

@@ -1 +1,16 @@
-sql = None
+import json
+import sqlite3
+from multiprocessing.pool import ThreadPool
+from helpers import databaseHelper
+
+with open("config.json", "r") as f:
+	config = json.load(f)
+
+app = None
+sql = None
+pool = ThreadPool(config["threads"])
+
+def new_sql():
+	db = sqlite3.connect("wayback.db")
+	db.row_factory = databaseHelper.dict_factory
+	return db

web/asyncTornado.py

@@ -1,6 +1,110 @@
 import tornado
 import tornado.web
 import tornado.gen
+from tornado.ioloop import IOLoop
 
+from constants import argumentTypes
+from objects import glob
+
+# Copied from https://zxq.co/ripple/ripple-python-common/src/branch/master/web/requestsManager.py
 class asyncRequestHandler(tornado.web.RequestHandler):
-    
+	"""
+	Tornado asynchronous request handler
+	create a class that extends this one (requestHelper.asyncRequestHandler)
+	use asyncGet() and asyncPost() instead of get() and post().
+	Done. I'm not kidding.
+	"""
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def get(self, *args, **kwargs):
+		try:
+			yield tornado.gen.Task(runBackground, (self.asyncGet, tuple(args), dict(kwargs)))
+		finally:
+			if not self._finished:
+				self.finish()
+
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def post(self, *args, **kwargs):
+		try:
+			yield tornado.gen.Task(runBackground, (self.asyncPost, tuple(args), dict(kwargs)))
+		finally:
+			if not self._finished:
+				self.finish()
+
+	def asyncGet(self, *args, **kwargs):
+		self.send_error(405)
+
+	def asyncPost(self, *args, **kwargs):
+		self.send_error(405)
+
+	def getRequestIP(self):
+		"""
+		Return CF-Connecting-IP (request IP when under cloudflare, you have to configure nginx to enable that)
+		If that fails, return X-Forwarded-For (request IP when not under Cloudflare)
+		if everything else fails, return remote IP
+
+		:return: Client IP address
+		"""
+		if "CF-Connecting-IP" in self.request.headers:
+			return self.request.headers.get("CF-Connecting-IP")
+		elif "X-Forwarded-For" in self.request.headers:
+			return self.request.headers.get("X-Forwarded-For")
+		else:
+			return self.request.remote_ip
+
+
+def runBackground(data, callback):
+	"""
+	Run a function in the background.
+	Used to handle multiple requests at the same time
+
+	:param data: (func, args, kwargs)
+	:param callback: function to call when `func` (data[0]) returns
+	:return:
+	"""
+	func, args, kwargs = data
+	def _callback(result):
+		IOLoop.instance().add_callback(lambda: callback(result))
+	glob.pool.apply_async(func, args, kwargs, _callback)
+
+def check_arguments(arguments, arguments_filter):
+	filter_pass = []
+	for k, v in arguments_filter.items():
+		if v == argumentTypes.optional:
+			filter_pass.append( arg_filter_and(arguments, k) )
+		elif v == argumentTypes.required:
+			filter_pass.append( arg_filter_require_all(arguments, k) )
+		elif v == argumentTypes.one_required:
+			filter_pass.append( arg_filter_first(arguments, k, False) )
+		elif v == argumentTypes.only_one:
+			filter_pass.append( arg_filter_only_one(arguments, k) )
+	return filter_pass
+
+def arg_filter_and(arguments, filter, can_false = False):
+	arg_filter = []
+	for i in filter:
+		if i in arguments:
+			arg_filter.append(i)
+	if can_false:
+		return arg_filter if len(arg_filter) else False
+	return arg_filter
+
+def arg_filter_require_all(arguments, required):
+	for i in required:
+		if i not in arguments:
+			return False
+	return required
+
+def arg_filter_only_one(arguments, required):
+	arg_filter = []
+	for i in required:
+		if i in arguments:
+			arg_filter.append(i)
+	return True if len(arg_filter) == 1 else False
+
+def arg_filter_first(arguments, filter, optional = True):
+	for i in filter:
+		if i in arguments:
+			return i
+	return optional