Try to send stdout/stderr to dashboard and fail at it

dashboard.html

@@ -1030,7 +1030,7 @@ JobsRenderer.prototype.handleData = function(data) {
 	var totalResponses = parseInt(getTotalResponses(jobData));
 	if(type == "download") {
 		var linesRendered = this._renderDownloadLine(data, info.logSegment);
-	} else if(type == "stdout") {
+	} else if(type == "stdout" || type == "stderr") {
 		var linesRendered = this._renderStdoutLine(data, info.logSegment, info, ident);
 	} else if(type == "ignore") {
 		var linesRendered = this._renderIgnoreLine(data, info.logSegment);

grab-site

@@ -68,4 +68,5 @@ GRAB_SITE_WORKING_DIR="$dir" PYTHONPATH="$self" ~/.local/bin/wpull3 \
 	--monitor-memory 10k \
 	--recursive \
 	--span-hosts-allow "$span_hosts_allow" \
+	--quiet \
 	"$url"

server.py

@@ -22,8 +22,13 @@ class MyServerProtocol(WebSocketServerProtocol):
 		print("{} disconnected from WebSocket server".format(self.peer))
 		self.factory.clients.remove(self)
 
+	def broadcastToDashboards(self, obj):
+		for client in self.factory.clients:
+			if client.mode == "dashboard":
+				client.sendMessage(json.dumps(obj).encode("utf-8"))
+
 	def onMessage(self, payload, isBinary):
-		print(payload)
+		##print(payload)
 		obj = json.loads(payload.decode('utf-8'))
 		type = obj["type"]
 		if type == "hello" and obj.get("mode"):
@@ -31,16 +36,20 @@ class MyServerProtocol(WebSocketServerProtocol):
 			if mode in ('dashboard', 'grabber'):
 				print("{} set mode {}".format(self.peer, mode))
 				self.mode = mode
-		elif type == "download" or type == "stdout":
-			for client in self.factory.clients:
-				if client.mode == "dashboard":
-					client.sendMessage(json.dumps({
-						"job_data": obj["job_data"],
-						"url": obj["url"],
-						"response_code": obj["response_code"],
-						"wget_code": obj["response_message"],
-						"type": type
-					}).encode("utf-8"))
+		elif type == "download":
+			self.broadcastToDashboards({
+				"type": type,
+				"job_data": obj["job_data"],
+				"url": obj["url"],
+				"response_code": obj["response_code"],
+				"wget_code": obj["response_message"]
+			})
+		elif type in ("stdout", "stderr"):
+			self.broadcastToDashboards({
+				"type": type,
+				"job_data": obj["job_data"],
+				"message": obj["message"]
+			})
 
 
 class MyServerFactory(WebSocketServerFactory):

wpull_hooks.py

@@ -1,5 +1,6 @@
 import re
 import os
+import sys
 import json
 import pprint
 import trollius as asyncio
@@ -7,6 +8,17 @@ from urllib.request import urlopen
 from autobahn.asyncio.websocket import WebSocketClientFactory, WebSocketClientProtocol
 from ignoracle import Ignoracle, parameterize_record_info
 
+real_stdout_write = sys.stdout.write
+real_stderr_write = sys.stderr.write
+
+def getJobData():
+	return {
+		"ident": ident,
+		"started_at": started_at,
+		"bytes_downloaded": stats["bytes_downloaded"],
+		"url": start_url
+	}
+
 class MyClientProtocol(WebSocketClientProtocol):
 	def onOpen(self):
 		self.factory.client = self
@@ -21,12 +33,7 @@ class MyClientProtocol(WebSocketClientProtocol):
 
 	def report(self, url, response_code, response_message):
 		self.sendMessage(json.dumps({
-			"job_data": {
-				"ident": ident,
-				"started_at": started_at,
-				"bytes_downloaded": stats["bytes_downloaded"],
-				"url": start_url,
-			},
+			"job_data": getJobData(),
 			"type": "download",
 			"url": url,
 			"response_code": response_code,
@@ -202,6 +209,34 @@ def handlePreResponse(urlInfo, url_record, response_info):
 	return wpull_hook.actions.NORMAL
 
 
+def stdoutWriteToBoth(message):
+	try:
+		real_stdout_write(message)
+		if wsFactory.client:
+			wsFactory.client.sendMessage({
+				"type": "stdout",
+				"job_data": getJobData(),
+				"message": message
+			})
+	except Exception as e:
+		real_stderr_write(str(e) + "\n")
+
+def stderrWriteToBoth(message):
+	try:
+		real_stderr_write(message)
+		if wsFactory.client:
+			wsFactory.client.sendMessage({
+				"type": "stderr",
+				"job_data": getJobData(),
+				"message": message
+			})
+	except Exception as e:
+		real_stderr_write(str(e) + "\n")
+
+sys.stdout.write = stdoutWriteToBoth
+sys.stderr.write = stderrWriteToBoth
+
+
 assert 2 in wpull_hook.callbacks.AVAILABLE_VERSIONS
 
 wpull_hook.callbacks.version = 2