Add OS X support
parent
cd893cb1e3
commit
0e38441234
16
README.md
16
README.md
|
@ -27,7 +27,7 @@ grab-site gives you
|
||||||
![dashboard screenshot](https://raw.githubusercontent.com/ludios/grab-site/master/images/dashboard.png)
|
![dashboard screenshot](https://raw.githubusercontent.com/ludios/grab-site/master/images/dashboard.png)
|
||||||
|
|
||||||
|
|
||||||
Installation
|
Install on Ubuntu
|
||||||
---
|
---
|
||||||
|
|
||||||
On Ubuntu 14.04.1 or newer:
|
On Ubuntu 14.04.1 or newer:
|
||||||
|
@ -38,6 +38,20 @@ pip3 install --user git+https://github.com/ludios/grab-site
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
Install on OS X
|
||||||
|
---
|
||||||
|
|
||||||
|
On OS X 10.10:
|
||||||
|
|
||||||
|
1. If xcode is not already installed, type `gcc` in Terminal; you will be prompted to install the command-line developer tools. Click 'Install'.
|
||||||
|
|
||||||
|
2. If Python 3 is not already installed, install Python 3.4.3 using the installer from https://www.python.org/downloads/release/python-343/
|
||||||
|
|
||||||
|
3. `pip3 install --user git+https://github.com/ludios/grab-site`
|
||||||
|
|
||||||
|
**Important usage note**: Use `~/Library/Python/3.4/bin/` instead of `~/.local/bin/` for all instructions below!
|
||||||
|
|
||||||
|
|
||||||
Usage
|
Usage
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
11
grab-site
11
grab-site
|
@ -5,7 +5,7 @@ set -e
|
||||||
url="$1"
|
url="$1"
|
||||||
id="$(python3 -c "import os, binascii; print(binascii.hexlify(os.urandom(16)).decode('utf-8'))")"
|
id="$(python3 -c "import os, binascii; print(binascii.hexlify(os.urandom(16)).decode('utf-8'))")"
|
||||||
# remove protocol, remove trailing slashes, convert slashes to "-"es
|
# remove protocol, remove trailing slashes, convert slashes to "-"es
|
||||||
dir="$(echo -n "$url" | sed -r 's,^(ftp|https?)://,,g' | sed -r 's,/+$,,g' | sed -r 's,[/\?&],-,g')-$(date +%F)-${id:0:8}"
|
dir="$(echo -nE "$url" | python3 -c "import sys, re; print(re.sub('[/\?&]', '-', sys.stdin.read().strip().split('://', 1)[1].rstrip('/')))")-$(date +%F)-${id:0:8}"
|
||||||
self=$(dirname "$0")
|
self=$(dirname "$0")
|
||||||
|
|
||||||
mkdir -p "$dir"
|
mkdir -p "$dir"
|
||||||
|
@ -45,11 +45,10 @@ for arg in "$@"; do
|
||||||
shift
|
shift
|
||||||
done
|
done
|
||||||
|
|
||||||
echo
|
echo -E "$id" > "$dir/id"
|
||||||
echo "$id" > "$dir/id"
|
echo -E "$url" > "$dir/start_url"
|
||||||
echo "$url" > "$dir/start_url"
|
echo -E "$concurrency" > "$dir/concurrency"
|
||||||
echo "$concurrency" > "$dir/concurrency"
|
echo -E "global,$igsets" > "$dir/igsets"
|
||||||
echo "global,$igsets" > "$dir/igsets"
|
|
||||||
touch "$dir/igoff"
|
touch "$dir/igoff"
|
||||||
touch "$dir/ignores"
|
touch "$dir/ignores"
|
||||||
|
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
__version__ = '0.2.0'
|
__version__ = '0.3.0'
|
||||||
|
|
|
@ -11,7 +11,11 @@ class DupesOnDisk(object):
|
||||||
try:
|
try:
|
||||||
self._env = lmdb.open(
|
self._env = lmdb.open(
|
||||||
filename,
|
filename,
|
||||||
writemap=True,
|
# Can't use writemap=True on OS X because it does not fully support sparse files
|
||||||
|
# https://acid.readthedocs.org/en/latest/engines.html
|
||||||
|
# Don't use writemap=True elsewhere because enormous sparse files get copied
|
||||||
|
# and make a non-sparse mess
|
||||||
|
writemap=False,
|
||||||
sync=False,
|
sync=False,
|
||||||
metasync=False,
|
metasync=False,
|
||||||
# http://lmdb.readthedocs.org/en/release/#lmdb.Environment
|
# http://lmdb.readthedocs.org/en/release/#lmdb.Environment
|
||||||
|
|
Loading…
Reference in New Issue