diff --git a/dashboard.html b/dashboard.html index ca79346..5e1178e 100644 --- a/dashboard.html +++ b/dashboard.html @@ -1030,7 +1030,7 @@ JobsRenderer.prototype.handleData = function(data) { var totalResponses = parseInt(getTotalResponses(jobData)); if(type == "download") { var linesRendered = this._renderDownloadLine(data, info.logSegment); - } else if(type == "stdout") { + } else if(type == "stdout" || type == "stderr") { var linesRendered = this._renderStdoutLine(data, info.logSegment, info, ident); } else if(type == "ignore") { var linesRendered = this._renderIgnoreLine(data, info.logSegment); diff --git a/grab-site b/grab-site index 72c17e9..c12072c 100755 --- a/grab-site +++ b/grab-site @@ -68,4 +68,5 @@ GRAB_SITE_WORKING_DIR="$dir" PYTHONPATH="$self" ~/.local/bin/wpull3 \ --monitor-memory 10k \ --recursive \ --span-hosts-allow "$span_hosts_allow" \ + --quiet \ "$url" diff --git a/server.py b/server.py index e190646..d596237 100644 --- a/server.py +++ b/server.py @@ -22,8 +22,13 @@ class MyServerProtocol(WebSocketServerProtocol): print("{} disconnected from WebSocket server".format(self.peer)) self.factory.clients.remove(self) + def broadcastToDashboards(self, obj): + for client in self.factory.clients: + if client.mode == "dashboard": + client.sendMessage(json.dumps(obj).encode("utf-8")) + def onMessage(self, payload, isBinary): - print(payload) + ##print(payload) obj = json.loads(payload.decode('utf-8')) type = obj["type"] if type == "hello" and obj.get("mode"): @@ -31,16 +36,20 @@ class MyServerProtocol(WebSocketServerProtocol): if mode in ('dashboard', 'grabber'): print("{} set mode {}".format(self.peer, mode)) self.mode = mode - elif type == "download" or type == "stdout": - for client in self.factory.clients: - if client.mode == "dashboard": - client.sendMessage(json.dumps({ - "job_data": obj["job_data"], - "url": obj["url"], - "response_code": obj["response_code"], - "wget_code": obj["response_message"], - "type": type - }).encode("utf-8")) + elif type == "download": + self.broadcastToDashboards({ + "type": type, + "job_data": obj["job_data"], + "url": obj["url"], + "response_code": obj["response_code"], + "wget_code": obj["response_message"] + }) + elif type in ("stdout", "stderr"): + self.broadcastToDashboards({ + "type": type, + "job_data": obj["job_data"], + "message": obj["message"] + }) class MyServerFactory(WebSocketServerFactory): diff --git a/wpull_hooks.py b/wpull_hooks.py index 5e467ee..dfd0a58 100644 --- a/wpull_hooks.py +++ b/wpull_hooks.py @@ -1,5 +1,6 @@ import re import os +import sys import json import pprint import trollius as asyncio @@ -7,6 +8,17 @@ from urllib.request import urlopen from autobahn.asyncio.websocket import WebSocketClientFactory, WebSocketClientProtocol from ignoracle import Ignoracle, parameterize_record_info +real_stdout_write = sys.stdout.write +real_stderr_write = sys.stderr.write + +def getJobData(): + return { + "ident": ident, + "started_at": started_at, + "bytes_downloaded": stats["bytes_downloaded"], + "url": start_url + } + class MyClientProtocol(WebSocketClientProtocol): def onOpen(self): self.factory.client = self @@ -21,12 +33,7 @@ class MyClientProtocol(WebSocketClientProtocol): def report(self, url, response_code, response_message): self.sendMessage(json.dumps({ - "job_data": { - "ident": ident, - "started_at": started_at, - "bytes_downloaded": stats["bytes_downloaded"], - "url": start_url, - }, + "job_data": getJobData(), "type": "download", "url": url, "response_code": response_code, @@ -202,6 +209,34 @@ def handlePreResponse(urlInfo, url_record, response_info): return wpull_hook.actions.NORMAL +def stdoutWriteToBoth(message): + try: + real_stdout_write(message) + if wsFactory.client: + wsFactory.client.sendMessage({ + "type": "stdout", + "job_data": getJobData(), + "message": message + }) + except Exception as e: + real_stderr_write(str(e) + "\n") + +def stderrWriteToBoth(message): + try: + real_stderr_write(message) + if wsFactory.client: + wsFactory.client.sendMessage({ + "type": "stderr", + "job_data": getJobData(), + "message": message + }) + except Exception as e: + real_stderr_write(str(e) + "\n") + +sys.stdout.write = stdoutWriteToBoth +sys.stderr.write = stderrWriteToBoth + + assert 2 in wpull_hook.callbacks.AVAILABLE_VERSIONS wpull_hook.callbacks.version = 2