Add cbor, gzip, lzma

README.md

@@ -6,10 +6,10 @@ This code is dirty for the moment. Experimental purpose only.
 
 Requires MineTest `blocks` table to be on PostgreSQL.
 
-Requires `python3-psycopg2` (which requires `libpq-dev`):
+Requires `python3-psycopg2` (which requires `libpq-dev`) and `cbor`:
 
     sudo apt install libpq-dev
-    sudo pip3 install psycopg2
+    sudo pip3 install psycopg2 cbor
 
 ## Configure
 
@@ -34,6 +34,17 @@ Response is a JSON list of the blocks of which position verifies these condition
         ]
     }
 
+Options:
+
+* `fmt` Response format: `json` (default), `cbor`
+* `cpr` Response compression: `none` (default), `gzip`, `lzma`
+
+Example:
+
+    curl "http://127.0.0.1:8060/x>=0/x<100/y>=0/y<100/z>=0/z<100/fmt/cbor/cpr/gzip" > test_gzip.cbor
+
+Prefer cbor+gzip for big imports. lzma is much slower than gzip but produces slightly smaller files. cbor is much smaller than json.
+
 ## License
 
 GNU AGPL 3.0

getblocks.py

@@ -1,22 +1,37 @@
 #!/usr/bin/env python3
 
-import json, psycopg2, re, socket, time
+import cbor, gzip, json, lzma, psycopg2, re, socket, time
 
 CONFIG_DB_CONNECT = "host=127.0.0.1 port=5432 user=minetest password=PASSWORD dbname=minetest-world"
 CONFIG_LISTEN = ("0.0.0.0", 8060)
 
 RECBUF = 1024
 
+AVAILABLE_FORMATS = {"json": "text/json", "cbor": "application/cbor"}
+AVAILABLE_COMPRESSIONS = {"none": None, "gzip": "gzip", "lzma": "lzma"}
+
 p_clen = re.compile("\r?\ncontent-length: *(\d+)\r?\n?", re.IGNORECASE)
 p_stmt = re.compile("^(x|y|z)(<|>|=|<=|>=)-?\d{1,5}$")
+encode = {
+	"json": lambda x: json.dumps(x).encode(),
+	"cbor": cbor.dumps
+}
+compress = {
+	"gzip": gzip.compress,
+	"lzma": lzma.compress
+}
 
-def send_response(client, code, resp):
+def send_response(client, code, resp, resp_format, resp_compression):
-	try:
+	content_raw = encode[resp_format](resp)
-		content_raw = json.dumps(resp).encode()
+	
-	except TypeError:
+	compression_str = ""
-		content_raw = json.dumps({"error": "non_ascii_resp"}).encode()
+	if AVAILABLE_COMPRESSIONS[resp_compression]:
-	mime = "text/json"
+		compression_str = "Content-Encoding: " + AVAILABLE_COMPRESSIONS[resp_compression] + "\r\n"
-	client.sendall(("HTTP/1.1 "+code+"\r\nContent-type: "+mime+"; charset=UTF-8\r\nAccess-Control-Allow-Origin: *\r\nContent-length: "+str(len(content_raw))+"\r\n\r\n").encode()+content_raw)
+		content_raw = compress[resp_compression](content_raw)
+	
+	mime = AVAILABLE_FORMATS[resp_format]
+	
+	client.sendall(("HTTP/1.1 "+code+"\r\nContent-type: "+mime+"; charset=UTF-8\r\n"+compression_str+"Access-Control-Allow-Origin: *\r\nContent-length: "+str(len(content_raw))+"\r\n\r\n").encode()+content_raw)
 	client.close()
 
 if __name__ == "__main__":
@@ -45,6 +60,8 @@ if __name__ == "__main__":
 		content = b""
 		content_len = 0
 		resp = {}
+		resp_format = "json"
+		resp_compression = "none"
 		lf = 0
 		while True:
 			raw = client.recv(RECBUF)
@@ -79,47 +96,79 @@ if __name__ == "__main__":
 		try:
 			url = httpreq[0].split(b" ")[1].decode().split("/")
 		except IndexError:
-			send_response(client, "400 Bad Request", {"error": "bad_http"})
+			send_response(client, "400 Bad Request", {"error": "bad_http"}, resp_format, resp_compression)
 			continue
 		while "" in url:
 			url.remove("")
 		urll = len(url)
 		
-		if len(url) > 6:
+		if urll > 32:
-			send_response(client, "400 Bad Request", {"error": "too_many_statements"})
+			send_response(client, "400 Bad Request", {"error": "too_many_args"}, resp_format, resp_compression)
-			continue
-		
-		if len(url) == 0:
-			send_response(client, "400 Bad Request", {"error": "no_statement"})
 			continue
 		
 		stmts = []
 		
 		bad = False
-		for stmt in url:
+		token = None
-			if not p_stmt.match(stmt):
+		for val in url:
+			if token:
+				if token == 1:
+					if not val in AVAILABLE_FORMATS:
+						bad = True
+						break
+					resp_format = val
+					token = None
+					continue
+				elif token == 2:
+					if not val in AVAILABLE_COMPRESSIONS:
+						bad = True
+						break
+					resp_compression = val
+					token = None
+					continue
 				bad = True
 				break
-			stmts.append("pos"+stmt)
+			
+			if val == "fmt":
+				token = 1
+				continue
+			elif val == "cpr":
+				token = 2
+				continue
+			
+			if not p_stmt.match(val):
+				bad = True
+				break
+			stmts.append("pos"+val)
 		
 		if bad:
-			send_response(client, "400 Bad Request", {"error": "bad_statement"})
+			send_response(client, "400 Bad Request", {"error": "bad_request"}, resp_format, resp_compression)
+			continue
+		
+		if len(stmts) > 6:
+			send_response(client, "400 Bad Request", {"error": "too_many_statements"}, resp_format, resp_compression)
+			continue
+		
+		if len(stmts) == 0:
+			send_response(client, "400 Bad Request", {"error": "no_statement"}, resp_format, resp_compression)
 			continue
 		
 		req = " AND ".join(stmts)
 		
 		cur = conn.cursor()
-		cur.execute("SELECT * FROM blocks WHERE "+req+" LIMIT 1000;")
+		cur.execute("SELECT * FROM blocks WHERE "+req+" LIMIT 1000000;")
 		resp["blocks"] = []
 		while True:
 			block = cur.fetchone()
 			if not block:
 				break
-			resp["blocks"].append([block[0], block[1], block[2], block[3].hex()])
+			resp["blocks"].append([block[0], block[1], block[2],
+				block[3].hex() if resp_format == "json" else block[3].tobytes()
+			])
 		
 		print(req)
 		
 		# Send response
-		send_response(client, "200 OK", resp)
+		send_response(client, "200 OK", resp, resp_format, resp_compression)
 		
 		time.sleep(.2)