Make online tests use localhost only, and factor out tests that require no networking at all

.travis.yml

@@ -13,19 +13,5 @@ install:
   - travis_retry pip install --process-dependency-links --no-binary --upgrade .
 
 script:
-  - grab-site --help
-  - grab-site --version
-  - grab-site --1 https://www.google.com/
-  - gs-dump-urls --help
-  - ls -l www.google.com*
-  - gs-dump-urls www.google.com*/wpull.db done
-  - grab-site --1 --permanent-error-status-codes=404 https://www.google.com/
-  - echo '.*' > ignores
-  - grab-site --import-ignores ignores https://www.google.com/
-  - grab-site --1 --id my-id --no-dupespotter --no-video --concurrent 3 https://www.google.com/ https://www.google.com/intl/en/ads/
-  - grab-site --1 -i https://gist.githubusercontent.com/ivan/1d437237746cf067e134/raw/47d2253c0caf4a6b964dc657840ddce22eb9eac7/grab-site%2520test%2520urls.txt
-  - curl https://gist.githubusercontent.com/ivan/1d437237746cf067e134/raw/47d2253c0caf4a6b964dc657840ddce22eb9eac7/grab-site%2520test%2520urls.txt > local-url-list
-  - grab-site --1 -i local-url-list
-  - grab-site --1 -i local-url-list --which-wpull-args-partial
-  - grab-site --1 -i local-url-list --which-wpull-command
-  - python3 -c 'import libgrabsite.server'
+  - ./tests/offline-tests
+  - ./tests/online-tests

tests/offline-tests

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail -o verbose
+
+grab-site --help
+grab-site --version
+gs-dump-urls --help
+python -c 'import libgrabsite.server'

tests/online-tests

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail -o verbose
+
+server_log=$(mktemp)
+python -u -m http.server 0 > "$server_log" &
+port=
+# Try until server starts up
+while [[ "$port" = "" ]]; do
+	port=$(grep -P -o 'port \d+' "$server_log" | cut -d ' ' -f 2 || true)
+	sleep 0.1
+done
+server="http://127.0.0.1:$port/"
+
+trash=$(mktemp -d)
+cd "$trash"
+
+nodnspython=--wpull-args=--no-skip-getaddrinfo
+
+grab-site --1 "$server"
+grab-site $nodnspython --1 "$server"
+ls -l 127.0.0.1*
+gs-dump-urls 127.0.0.1*/wpull.db done
+grab-site $nodnspython --1 --permanent-error-status-codes=404 "$server"
+echo '.*' > ignores
+grab-site $nodnspython --import-ignores ignores "$server"
+grab-site $nodnspython --1 --id my-id --no-dupespotter --no-video --concurrent 3 "$server" "$server/another"
+# TODO: test -i with remote URL list
+echo "$server" > local-url-list
+grab-site $nodnspython --1 -i local-url-list
+grab-site $nodnspython --1 -i local-url-list --which-wpull-args-partial
+grab-site $nodnspython --1 -i local-url-list --which-wpull-command
+
+# kill http.server
+kill $!