diff --git a/dashboard.html b/dashboard.html
index ca79346..5e1178e 100644
--- a/dashboard.html
+++ b/dashboard.html
@@ -1030,7 +1030,7 @@ JobsRenderer.prototype.handleData = function(data) {
var totalResponses = parseInt(getTotalResponses(jobData));
if(type == "download") {
var linesRendered = this._renderDownloadLine(data, info.logSegment);
- } else if(type == "stdout") {
+ } else if(type == "stdout" || type == "stderr") {
var linesRendered = this._renderStdoutLine(data, info.logSegment, info, ident);
} else if(type == "ignore") {
var linesRendered = this._renderIgnoreLine(data, info.logSegment);
diff --git a/grab-site b/grab-site
index 72c17e9..c12072c 100755
--- a/grab-site
+++ b/grab-site
@@ -68,4 +68,5 @@ GRAB_SITE_WORKING_DIR="$dir" PYTHONPATH="$self" ~/.local/bin/wpull3 \
--monitor-memory 10k \
--recursive \
--span-hosts-allow "$span_hosts_allow" \
+ --quiet \
"$url"
diff --git a/server.py b/server.py
index e190646..d596237 100644
--- a/server.py
+++ b/server.py
@@ -22,8 +22,13 @@ class MyServerProtocol(WebSocketServerProtocol):
print("{} disconnected from WebSocket server".format(self.peer))
self.factory.clients.remove(self)
+ def broadcastToDashboards(self, obj):
+ for client in self.factory.clients:
+ if client.mode == "dashboard":
+ client.sendMessage(json.dumps(obj).encode("utf-8"))
+
def onMessage(self, payload, isBinary):
- print(payload)
+ ##print(payload)
obj = json.loads(payload.decode('utf-8'))
type = obj["type"]
if type == "hello" and obj.get("mode"):
@@ -31,16 +36,20 @@ class MyServerProtocol(WebSocketServerProtocol):
if mode in ('dashboard', 'grabber'):
print("{} set mode {}".format(self.peer, mode))
self.mode = mode
- elif type == "download" or type == "stdout":
- for client in self.factory.clients:
- if client.mode == "dashboard":
- client.sendMessage(json.dumps({
- "job_data": obj["job_data"],
- "url": obj["url"],
- "response_code": obj["response_code"],
- "wget_code": obj["response_message"],
- "type": type
- }).encode("utf-8"))
+ elif type == "download":
+ self.broadcastToDashboards({
+ "type": type,
+ "job_data": obj["job_data"],
+ "url": obj["url"],
+ "response_code": obj["response_code"],
+ "wget_code": obj["response_message"]
+ })
+ elif type in ("stdout", "stderr"):
+ self.broadcastToDashboards({
+ "type": type,
+ "job_data": obj["job_data"],
+ "message": obj["message"]
+ })
class MyServerFactory(WebSocketServerFactory):
diff --git a/wpull_hooks.py b/wpull_hooks.py
index 5e467ee..dfd0a58 100644
--- a/wpull_hooks.py
+++ b/wpull_hooks.py
@@ -1,5 +1,6 @@
import re
import os
+import sys
import json
import pprint
import trollius as asyncio
@@ -7,6 +8,17 @@ from urllib.request import urlopen
from autobahn.asyncio.websocket import WebSocketClientFactory, WebSocketClientProtocol
from ignoracle import Ignoracle, parameterize_record_info
+real_stdout_write = sys.stdout.write
+real_stderr_write = sys.stderr.write
+
+def getJobData():
+ return {
+ "ident": ident,
+ "started_at": started_at,
+ "bytes_downloaded": stats["bytes_downloaded"],
+ "url": start_url
+ }
+
class MyClientProtocol(WebSocketClientProtocol):
def onOpen(self):
self.factory.client = self
@@ -21,12 +33,7 @@ class MyClientProtocol(WebSocketClientProtocol):
def report(self, url, response_code, response_message):
self.sendMessage(json.dumps({
- "job_data": {
- "ident": ident,
- "started_at": started_at,
- "bytes_downloaded": stats["bytes_downloaded"],
- "url": start_url,
- },
+ "job_data": getJobData(),
"type": "download",
"url": url,
"response_code": response_code,
@@ -202,6 +209,34 @@ def handlePreResponse(urlInfo, url_record, response_info):
return wpull_hook.actions.NORMAL
+def stdoutWriteToBoth(message):
+ try:
+ real_stdout_write(message)
+ if wsFactory.client:
+ wsFactory.client.sendMessage({
+ "type": "stdout",
+ "job_data": getJobData(),
+ "message": message
+ })
+ except Exception as e:
+ real_stderr_write(str(e) + "\n")
+
+def stderrWriteToBoth(message):
+ try:
+ real_stderr_write(message)
+ if wsFactory.client:
+ wsFactory.client.sendMessage({
+ "type": "stderr",
+ "job_data": getJobData(),
+ "message": message
+ })
+ except Exception as e:
+ real_stderr_write(str(e) + "\n")
+
+sys.stdout.write = stdoutWriteToBoth
+sys.stderr.write = stderrWriteToBoth
+
+
assert 2 in wpull_hook.callbacks.AVAILABLE_VERSIONS
wpull_hook.callbacks.version = 2