Archiveteam
/
grab-site
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generate a grab id and put in the dir name; add some temporary print debugging

master
Ivan Kozik 2015-07-18 01:06:56 +00:00
parent 5621863b09
commit db21e530e2
2 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
grab-site
View File

@ -3,8 +3,9 @@
set -e
url="$1"
id="$(python3 -c "import os, binascii; print(binascii.hexlify(os.urandom(16)).decode('utf-8'))")"
# remove protocol, remove trailing slashes, convert slashes to "-"es
dir="$(echo -n "$url" | sed -r 's,^(ftp|https?)://,,g' | sed -r 's,/+$,,g' | sed -r 's,[/\?&],-,g')-$(date +%F)"
dir="$(echo -n "$url" | sed -r 's,^(ftp|https?)://,,g' | sed -r 's,/+$,,g' | sed -r 's,[/\?&],-,g')-$(date +%F)-${id:0:8}"
self=$(dirname "$0")
mkdir -p "$dir"
@ -26,6 +27,8 @@ for arg in "$@"; do
shift
done
echo
echo "$id" > "$dir/id"
echo "global,$ignore_sets" > "$dir/ignore_sets"
touch "$dir/ignores"

15
wpull_hooks.py
View File

@ -107,10 +107,22 @@ def accept_url(url_info, record_info, verdict, reasons):
return verdict
def handle_response(url_info, record_info, error_info=None, http_info=None):
def handle_result(url_info, record_info, error_info=None, http_info=None):
print("url_info", url_info)
print("record_info", record_info)
print("error_info", error_info)
print("http_info", http_info)
wsFactory.client.report(url_info['url'])
def handle_response(url_info, record_info, http_info):
return handle_result(url_info, record_info, http_info=http_info)
def handle_error(url_info, record_info, error_info):
return handle_result(url_info, record_info, error_info=error_info)
# Regular expressions for server headers go here
ICY_FIELD_PATTERN = re.compile('Icy-|Ice-|X-Audiocast-')
ICY_VALUE_PATTERN = re.compile('icecast', re.IGNORECASE)
@ -145,4 +157,5 @@ assert 2 in wpull_hook.callbacks.AVAILABLE_VERSIONS
wpull_hook.callbacks.version = 2
wpull_hook.callbacks.accept_url = accept_url
wpull_hook.callbacks.handle_response = handle_response
wpull_hook.callbacks.handle_error = handle_error
wpull_hook.callbacks.handle_pre_response = handle_pre_response