65 lines
1.6 KiB
Bash
Executable File
65 lines
1.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e
|
|
|
|
url="$1"
|
|
# remove protocol, remove trailing slashes, convert slashes to "-"es
|
|
dir="$(echo -n "$url" | sed -r 's,^(ftp|https?)://,,g' | sed -r 's,/+$,,g' | sed -r 's,[/\?&],-,g')-$(date +%F)"
|
|
self=$(dirname "$0")
|
|
|
|
mkdir -p "$dir"
|
|
|
|
span_hosts_allow="page-requisites,linked-pages"
|
|
|
|
for arg in "$@"; do
|
|
case $arg in
|
|
--ignore-sets=*)
|
|
ignore_sets="${arg#*=}"
|
|
;;
|
|
--no-offsite-links)
|
|
span_hosts_allow="page-requisites"
|
|
;;
|
|
*)
|
|
# unknown option
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
echo "global,$ignore_sets" > "$dir/ignore_sets"
|
|
touch "$dir/ignores"
|
|
|
|
# Note: we use the default html5lib parser instead of the lxml that ArchiveBot uses
|
|
# html5lib is slower, but is better at parsing and doesn't (rarely) corrupt the heap like lxml
|
|
|
|
HOOK_SETTINGS_DIR="$dir" PYTHONPATH="$self" ~/.local/bin/wpull3 \
|
|
-U "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:37.0) Gecko/20100101 Firefox/37.0" \
|
|
--header="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" \
|
|
--header="Accept-Language: en-US,en;q=0.5" \
|
|
-o "$dir/wpull.log" \
|
|
--database "$dir/wpull.db" \
|
|
--plugin-script "$self/plugin.py" \
|
|
--python-script "$self/wpull_hooks.py" \
|
|
--plugin-args " --dupes-db $dir/dupes_db" \
|
|
--save-cookies "$dir/cookies.txt" \
|
|
--no-check-certificate \
|
|
--delete-after \
|
|
--no-robots \
|
|
--page-requisites \
|
|
--no-parent \
|
|
--sitemaps \
|
|
--inet4-only \
|
|
--timeout 20 \
|
|
--tries 3 \
|
|
--waitretry 5 \
|
|
--warc-file "$dir/$dir" \
|
|
--warc-max-size 5368709120 \
|
|
--debug-manhole \
|
|
--strip-session-id \
|
|
--escaped-fragment \
|
|
--monitor-disk 50m \
|
|
--monitor-memory 10k \
|
|
--recursive \
|
|
--span-hosts-allow "$span_hosts_allow" \
|
|
"$url"
|