Archiveteam/grab-site
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Report networking errors correctly

Browse Source
This commit is contained in:
Ivan Kozik 2015-07-18 09:49:03 +00:00
parent 8a8ea70a7d
commit 66d869a545
2 changed files with 22 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dashboard.html
View File

@ -915,17 +915,18 @@ JobsRenderer.prototype._createLogContainer = function(jobData) {
}
JobsRenderer.prototype._renderDownloadLine = function(data, logSegment) {
if(data["response_code"] >= 400 && data["response_code"] < 500) {
var code = data["response_code"];
if(code >= 400 && code < 500) {
var attrs = {"className": "line-warning", "href": data["url"]};
} else if(data["response_code"] >= 500) {
} else if(code === 0 || code >= 500) {
var attrs = {"className": "line-error", "href": data["url"]};
} else if(data["response_code"] && data["response_code"] >= 300 && data["response_code"] < 400) {
} else if(code && code >= 300 && code < 400) {
var attrs = {"className": "line-redirect", "href": data["url"]};
} else {
var attrs = {"className": "line-normal", "href": data["url"]};
}
logSegment.appendChild(
h("a", attrs, data["response_code"] + " " + data["wget_code"] + " " + data["url"])
h("a", attrs, code + " " + data["wget_code"] + " " + data["url"])
);
return 1;
};
@ -935,17 +936,18 @@ JobsRenderer.prototype._renderDownloadLine = function(data, logSegment) {
* left or right of the URL.
*/
JobsRenderer.prototype._moreDomRenderDownloadLine = function(data, logSegment) {
if(data["response_code"] >= 400 && data["response_code"] < 500) {
var code = data["response_code"];
if(code >= 400 && code < 500) {
var attrs = Reusable.obj_className_line_warning;
} else if(data["response_code"] >= 500) {
} else if(code === 0 || code >= 500) {
var attrs = Reusable.obj_className_line_error;
} else if(data["response_code"] && data["response_code"] >= 300 && data["response_code"] < 400) {
} else if(code && code >= 300 && code < 400) {
var attrs = Reusable.obj_className_line_redirect;
} else {
var attrs = Reusable.obj_className_line_normal;
}
logSegment.appendChild(h("div", attrs, [
data["response_code"] + " " + data["wget_code"] + " ",
code + " " + data["wget_code"] + " ",
h("a", {"href": data["url"], "className": "log-url"}, data["url"])
]));
return 1;

17
wpull_hooks.py
View File

@ -190,23 +190,30 @@ def handleResult(urlInfo, recordInfo, errorInfo={}, httpInfo={}):
updateIgoffInJobData()
response_code = 0
if httpInfo.get("response_code"):
response_code = str(httpInfo["response_code"])
if len(response_code) == 3 and response_code[0] in "12345":
jobData["r%sxx" % response_code[0]] += 1
response_code = httpInfo.get("response_code")
response_code_str = str(httpInfo["response_code"])
if len(response_code_str) == 3 and response_code_str[0] in "12345":
jobData["r%sxx" % response_code_str[0]] += 1
if httpInfo.get("body"):
jobData["bytes_downloaded"] += httpInfo["body"]["content_size"]
stop = shouldStop()
response_message = httpInfo.get("response_message")
if errorInfo:
response_code = 0
response_message = errorInfo["error"]
if wsFactory.client:
wsFactory.client.sendObject({
"type": "download",
"job_data": jobData,
"url": urlInfo["url"],
"response_code": httpInfo.get("response_code"),
"response_message": httpInfo.get("response_message"),
"response_code": response_code,
"response_message": response_message,
})
if stop: