Add cbor, gzip, lzma
This commit is contained in:
tuxmain
parent
60483cafc8
commit
6385e11a76
15
README.md
15
README.md
@ -6,10 +6,10 @@ This code is dirty for the moment. Experimental purpose only.
|
||||
|
||||
Requires MineTest `blocks` table to be on PostgreSQL.
|
||||
|
||||
Requires `python3-psycopg2` (which requires `libpq-dev`):
|
||||
Requires `python3-psycopg2` (which requires `libpq-dev`) and `cbor`:
|
||||
|
||||
sudo apt install libpq-dev
|
||||
sudo pip3 install psycopg2
|
||||
sudo pip3 install psycopg2 cbor
|
||||
|
||||
## Configure
|
||||
|
||||
@ -34,6 +34,17 @@ Response is a JSON list of the blocks of which position verifies these condition
|
||||
]
|
||||
}
|
||||
|
||||
Options:
|
||||
|
||||
* `fmt` Response format: `json` (default), `cbor`
|
||||
* `cpr` Response compression: `none` (default), `gzip`, `lzma`
|
||||
|
||||
Example:
|
||||
|
||||
curl "http://127.0.0.1:8060/x>=0/x<100/y>=0/y<100/z>=0/z<100/fmt/cbor/cpr/gzip" > test_gzip.cbor
|
||||
|
||||
Prefer cbor+gzip for big imports. lzma is much slower than gzip but produces slightly smaller files. cbor is much smaller than json.
|
||||
|
||||
## License
|
||||
|
||||
GNU AGPL 3.0
|
||||
|
||||
93
getblocks.py
93
getblocks.py
@ -1,22 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json, psycopg2, re, socket, time
|
||||
import cbor, gzip, json, lzma, psycopg2, re, socket, time
|
||||
|
||||
CONFIG_DB_CONNECT = "host=127.0.0.1 port=5432 user=minetest password=PASSWORD dbname=minetest-world"
|
||||
CONFIG_LISTEN = ("0.0.0.0", 8060)
|
||||
|
||||
RECBUF = 1024
|
||||
|
||||
AVAILABLE_FORMATS = {"json": "text/json", "cbor": "application/cbor"}
|
||||
AVAILABLE_COMPRESSIONS = {"none": None, "gzip": "gzip", "lzma": "lzma"}
|
||||
|
||||
p_clen = re.compile("\r?\ncontent-length: *(\d+)\r?\n?", re.IGNORECASE)
|
||||
p_stmt = re.compile("^(x|y|z)(<|>|=|<=|>=)-?\d{1,5}$")
|
||||
encode = {
|
||||
"json": lambda x: json.dumps(x).encode(),
|
||||
"cbor": cbor.dumps
|
||||
}
|
||||
compress = {
|
||||
"gzip": gzip.compress,
|
||||
"lzma": lzma.compress
|
||||
}
|
||||
|
||||
def send_response(client, code, resp):
|
||||
try:
|
||||
content_raw = json.dumps(resp).encode()
|
||||
except TypeError:
|
||||
content_raw = json.dumps({"error": "non_ascii_resp"}).encode()
|
||||
mime = "text/json"
|
||||
client.sendall(("HTTP/1.1 "+code+"\r\nContent-type: "+mime+"; charset=UTF-8\r\nAccess-Control-Allow-Origin: *\r\nContent-length: "+str(len(content_raw))+"\r\n\r\n").encode()+content_raw)
|
||||
def send_response(client, code, resp, resp_format, resp_compression):
|
||||
content_raw = encode[resp_format](resp)
|
||||
|
||||
compression_str = ""
|
||||
if AVAILABLE_COMPRESSIONS[resp_compression]:
|
||||
compression_str = "Content-Encoding: " + AVAILABLE_COMPRESSIONS[resp_compression] + "\r\n"
|
||||
content_raw = compress[resp_compression](content_raw)
|
||||
|
||||
mime = AVAILABLE_FORMATS[resp_format]
|
||||
|
||||
client.sendall(("HTTP/1.1 "+code+"\r\nContent-type: "+mime+"; charset=UTF-8\r\n"+compression_str+"Access-Control-Allow-Origin: *\r\nContent-length: "+str(len(content_raw))+"\r\n\r\n").encode()+content_raw)
|
||||
client.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
@ -45,6 +60,8 @@ if __name__ == "__main__":
|
||||
content = b""
|
||||
content_len = 0
|
||||
resp = {}
|
||||
resp_format = "json"
|
||||
resp_compression = "none"
|
||||
lf = 0
|
||||
while True:
|
||||
raw = client.recv(RECBUF)
|
||||
@ -79,47 +96,79 @@ if __name__ == "__main__":
|
||||
try:
|
||||
url = httpreq[0].split(b" ")[1].decode().split("/")
|
||||
except IndexError:
|
||||
send_response(client, "400 Bad Request", {"error": "bad_http"})
|
||||
send_response(client, "400 Bad Request", {"error": "bad_http"}, resp_format, resp_compression)
|
||||
continue
|
||||
while "" in url:
|
||||
url.remove("")
|
||||
urll = len(url)
|
||||
|
||||
if len(url) > 6:
|
||||
send_response(client, "400 Bad Request", {"error": "too_many_statements"})
|
||||
continue
|
||||
|
||||
if len(url) == 0:
|
||||
send_response(client, "400 Bad Request", {"error": "no_statement"})
|
||||
if urll > 32:
|
||||
send_response(client, "400 Bad Request", {"error": "too_many_args"}, resp_format, resp_compression)
|
||||
continue
|
||||
|
||||
stmts = []
|
||||
|
||||
bad = False
|
||||
for stmt in url:
|
||||
if not p_stmt.match(stmt):
|
||||
token = None
|
||||
for val in url:
|
||||
if token:
|
||||
if token == 1:
|
||||
if not val in AVAILABLE_FORMATS:
|
||||
bad = True
|
||||
break
|
||||
stmts.append("pos"+stmt)
|
||||
resp_format = val
|
||||
token = None
|
||||
continue
|
||||
elif token == 2:
|
||||
if not val in AVAILABLE_COMPRESSIONS:
|
||||
bad = True
|
||||
break
|
||||
resp_compression = val
|
||||
token = None
|
||||
continue
|
||||
bad = True
|
||||
break
|
||||
|
||||
if val == "fmt":
|
||||
token = 1
|
||||
continue
|
||||
elif val == "cpr":
|
||||
token = 2
|
||||
continue
|
||||
|
||||
if not p_stmt.match(val):
|
||||
bad = True
|
||||
break
|
||||
stmts.append("pos"+val)
|
||||
|
||||
if bad:
|
||||
send_response(client, "400 Bad Request", {"error": "bad_statement"})
|
||||
send_response(client, "400 Bad Request", {"error": "bad_request"}, resp_format, resp_compression)
|
||||
continue
|
||||
|
||||
if len(stmts) > 6:
|
||||
send_response(client, "400 Bad Request", {"error": "too_many_statements"}, resp_format, resp_compression)
|
||||
continue
|
||||
|
||||
if len(stmts) == 0:
|
||||
send_response(client, "400 Bad Request", {"error": "no_statement"}, resp_format, resp_compression)
|
||||
continue
|
||||
|
||||
req = " AND ".join(stmts)
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT * FROM blocks WHERE "+req+" LIMIT 1000;")
|
||||
cur.execute("SELECT * FROM blocks WHERE "+req+" LIMIT 1000000;")
|
||||
resp["blocks"] = []
|
||||
while True:
|
||||
block = cur.fetchone()
|
||||
if not block:
|
||||
break
|
||||
resp["blocks"].append([block[0], block[1], block[2], block[3].hex()])
|
||||
resp["blocks"].append([block[0], block[1], block[2],
|
||||
block[3].hex() if resp_format == "json" else block[3].tobytes()
|
||||
])
|
||||
|
||||
print(req)
|
||||
|
||||
# Send response
|
||||
send_response(client, "200 OK", resp)
|
||||
send_response(client, "200 OK", resp, resp_format, resp_compression)
|
||||
|
||||
time.sleep(.2)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user