Sunpy/Philips_TV
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Yes... I forgot to commit during writeups etc.

Browse Source
This commit is contained in:
Emily
2020-04-29 16:58:59 +02:00
parent 4c755a99a8
commit a35bdff914
29 changed files with 1217 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
files/12_dec_2018/README.md Normal file
View File

@@ -0,0 +1,2 @@
# 12 December 2018
Not my greatest python work, but for a quick thing that I never had intended to release; it did its job.

8
files/12_dec_2018/files/config.json Normal file
View File

@@ -0,0 +1,8 @@
{
"ws": "192.168.2.60:9222/devtools/page/4133cee9-ac82-4404-b886-9032aee57018",
"root": "file:///",
"max-filesize": "16000000",
"output": "filesystem/sys",
"file-meta": "filesystem-meta/sys",
"output-meta": "filesystem-meta/sys"
}

312
files/12_dec_2018/files/file_browser.js Normal file
View File

File diff suppressed because one or more lines are too long

157
files/12_dec_2018/files/scrape_files.py Normal file
View File

@@ -0,0 +1,157 @@
import os
import json
from websocket import create_connection
import glob
import base64
with open("config.json", "r") as f:
config = json.load(f)
ws = create_connection("ws://{}".format(config["ws"]))
javascript_func = """
function load(file) {
let request = new XMLHttpRequest();
request.open('GET', file, false);
request.overrideMimeType('text\/plain; charset=x-user-defined');
request.send();
let data = request.responseText;
let output = "";
for (let i = 0; i < data.length; i++) {
let n = (data.charCodeAt(i) & 0xff).toString(16);
if (n.length == 1)
n = "0" + n;
output += n;
}
return output;
}
"""
struct = {
"id": 0,
"method": "Unset"
}
struct_params = struct.copy()
struct_params["params"] = {}
struct_enable_runtime = struct.copy()
struct_enable_runtime["method"] = "Runtime.enable"
struct_evaluate = struct_params.copy()
struct_evaluate["method"] = "Runtime.evaluate"
struct_evaluate["params"]["expression"] = "console.log(\"missing code\")"
struct_evaluate["params"]["objectGroup"] = "console"
struct_evaluate["params"]["includeCommandLineAPI"] = True
struct_evaluate["params"]["silent"] = False
struct_evaluate["params"]["contextId"] = 0
struct_evaluate["params"]["returnByValue"] = False
struct_evaluate["params"]["generatePreview"] = True
struct_evaluate["params"]["userGesture"] = True
struct_evaluate["params"]["awaitPromise"] = False
struct_compile_script = struct_params.copy()
struct_compile_script["method"] = "Runtime.compileScript"
struct_compile_script["params"]["expression"] = "console.log(\"missing code\")"
struct_compile_script["params"]["sourceURL"] = ""
struct_compile_script["params"]["persistScript"] = False
struct_compile_script["params"]["executionContextId"] = 0
def evalJavascript(script):
print("Executing: {}".format(script))
compile_msg = struct_compile_script.copy()
compile_msg["params"]["expression"] = script
send(compile_msg)
eval_msg = struct_evaluate.copy()
eval_msg["params"]["expression"] = script
data = send(eval_msg)
if data["result"]["result"]["type"] != "string":
return "undefined"
return data["result"]["result"]["value"]
def send(data, everything = False):
glob.message_id += 1
if "params" in data and "contextId" in data["params"]:
data["params"]["contextId"] = glob.context
if "params" in data and "executionContextId" in data["params"]:
data["params"]["executionContextId"] = glob.context
data["id"] = glob.message_id
ws.send(json.dumps(data))
if everything:
recv = [json.loads(ws.recv())]
while "id" not in recv[-1] or recv[-1]["id"] != data["id"]:
recv.append(json.loads(ws.recv()))
else:
recv = json.loads(ws.recv())
while "id" not in recv or recv["id"] != data["id"]:
recv = json.loads(ws.recv())
return recv
runtime = send(struct_enable_runtime, True)
for runtime_data in runtime:
if "method" in runtime_data and runtime_data["method"] == "Runtime.executionContextCreated" and "params" in runtime_data and "context" in runtime_data["params"] and "auxData" in runtime_data["params"]["context"] and "isDefault" in runtime_data["params"]["context"]["auxData"] and runtime_data["params"]["context"]["auxData"]["isDefault"]:
glob.context = runtime_data["params"]["context"]["id"]
print("Context set to ID: {}".format(glob.context))
evalJavascript(javascript_func)
remote_path = config["root"]
scrape_path = config["output"]
meta_path = config["file-meta"]
path_offset = []
def dig():
d = os.listdir(meta_path + "/" + "/".join(path_offset))
for p in d:
if os.path.isdir(meta_path + "/" + "/".join(path_offset) + "/" + p):
path_offset.append(p)
if not os.path.exists(scrape_path + "/" + "/".join(path_offset)):
os.makedirs(scrape_path + "/" + "/".join(path_offset))
dig()
path_offset.pop()
else:
#if os.path.exists(scrape_path + "/" + "/".join(path_offset) + "/" + p):
# print("Skipping existing file {}".format(scrape_path + "/" + "/".join(path_offset) + "/" + p))
# continue
#print("Reading file: {}".format(meta_path + "/" + "/".join(path_offset) + "/" + p))
with open(meta_path + "/" + "/".join(path_offset) + "/" + p, "r") as f:
meta_data = int(f.readline())
#if meta_data > int(config["max-filesize"]) or meta_data == 0:
# open(scrape_path + "/" + "/".join(path_offset) + "/" + p + "._maxsize", "w").close()
# continue
if meta_data < 16000000:
#print("Skipping...")
continue
print("Downloading... (expecting {} B)".format(meta_data))
data = evalJavascript("load(\"{}\")".format(remote_path + "/".join(path_offset) + "/" + p))
if data == "undefined":
with open(scrape_path + "/" + "/".join(path_offset) + "/" + p, "w") as f:
f.write("Error?")
continue
with open(scrape_path + "/" + "/".join(path_offset) + "/" + p, "wb") as f:
f.write(bytes([int(data[i:i + 2], 16) for i in range(0, len(data), 2)]))
dig()
ws.close()

153
files/12_dec_2018/files/scrape_struct.py Normal file
View File

@@ -0,0 +1,153 @@
import os
import json
from websocket import create_connection
import glob
with open("config.json", "r") as f:
config = json.load(f)
ws = create_connection("ws://{}".format(config["ws"]))
javascript_func = """
function load(file) {
let request = new XMLHttpRequest();
request.open('GET', file, false);
request.send();
return request.responseText;
}
"""
struct = {
"id": 0,
"method": "Unset"
}
struct_params = struct.copy()
struct_params["params"] = {}
struct_enable_runtime = struct.copy()
struct_enable_runtime["method"] = "Runtime.enable"
struct_evaluate = struct_params.copy()
struct_evaluate["method"] = "Runtime.evaluate"
struct_evaluate["params"]["expression"] = "console.log(\"missing code\")"
struct_evaluate["params"]["objectGroup"] = "console"
struct_evaluate["params"]["includeCommandLineAPI"] = True
struct_evaluate["params"]["silent"] = False
struct_evaluate["params"]["contextId"] = 0
struct_evaluate["params"]["returnByValue"] = False
struct_evaluate["params"]["generatePreview"] = True
struct_evaluate["params"]["userGesture"] = True
struct_evaluate["params"]["awaitPromise"] = False
struct_compile_script = struct_params.copy()
struct_compile_script["method"] = "Runtime.compileScript"
struct_compile_script["params"]["expression"] = "console.log(\"missing code\")"
struct_compile_script["params"]["sourceURL"] = ""
struct_compile_script["params"]["persistScript"] = False
struct_compile_script["params"]["executionContextId"] = 0
def evalJavascript(script):
print("Executing: {}".format(script))
compile_msg = struct_compile_script.copy()
compile_msg["params"]["expression"] = script
send(compile_msg)
eval_msg = struct_evaluate.copy()
eval_msg["params"]["expression"] = script
data = send(eval_msg)
if data["result"]["result"]["type"] != "string":
return "undefined"
return data["result"]["result"]["value"]
def send(data, everything = False):
glob.message_id += 1
if "params" in data and "contextId" in data["params"]:
data["params"]["contextId"] = glob.context
if "params" in data and "executionContextId" in data["params"]:
data["params"]["executionContextId"] = glob.context
data["id"] = glob.message_id
ws.send(json.dumps(data))
if everything:
recv = [json.loads(ws.recv())]
while "id" not in recv[-1] or recv[-1]["id"] != data["id"]:
recv.append(json.loads(ws.recv()))
else:
recv = json.loads(ws.recv())
while "id" not in recv or recv["id"] != data["id"]:
recv = json.loads(ws.recv())
return recv
def make_dir(relative_path):
if not os.path.exists(scrape_path + "/" + "/".join(path_offset) + "/" + relative_path):
os.makedirs(scrape_path + "/" + "/".join(path_offset) + "/" + relative_path)
def make_file(relative_path, size):
with open(scrape_path + "/" + "/".join(path_offset) + "/" + relative_path, "w") as f:
f.write(size)
def isRecursive(arr):
return len(set(arr)) != len(arr)
def handleResponse(code):
start = code.find("<script>")
while start != -1:
code = code[start + 8:]
start = code.find("<script>")
if not code.startswith("addRow("):
continue
focus = code.find(");</script>")
if focus == -1:
continue
args = code[7:focus].split(",")
for n in range(len(args)):
args[n] = args[n].strip('"')
name = args[0]
url = args[1]
is_dir = bool(int(args[2]))
size = int(args[3])
if url == "..":
continue
if is_dir:
make_dir(url)
#if url in ["proc", "sys"]:
# continue
path_offset.append(url)
if not isRecursive(path_offset):
handleResponse( evalJavascript("load(\"{}\")".format(remote_path + "/".join(path_offset))) )
else:
print("{} is recursive -> skipping!".format(remote_path + "/".join(path_offset)))
path_offset.pop()
else:
make_file(url, str(size))
runtime = send(struct_enable_runtime, True)
for runtime_data in runtime:
if "method" in runtime_data and runtime_data["method"] == "Runtime.executionContextCreated" and "params" in runtime_data and "context" in runtime_data["params"] and "auxData" in runtime_data["params"]["context"] and "isDefault" in runtime_data["params"]["context"]["auxData"] and runtime_data["params"]["context"]["auxData"]["isDefault"]:
glob.context = runtime_data["params"]["context"]["id"]
print("Context set to ID: {}".format(glob.context))
evalJavascript(javascript_func)
remote_path = config["root"]
scrape_path = config["output-meta"]
path_offset = []
handleResponse( evalJavascript("load(\"{}\")".format(remote_path + "/".join(path_offset))) )
ws.close()