Add OS X support

README.md

@@ -27,7 +27,7 @@ grab-site gives you
 ![dashboard screenshot](https://raw.githubusercontent.com/ludios/grab-site/master/images/dashboard.png)
 
 
-Installation
+Install on Ubuntu
 ---
 
 On Ubuntu 14.04.1 or newer:
@@ -38,6 +38,20 @@ pip3 install --user git+https://github.com/ludios/grab-site
 ```
 
 
+Install on OS X
+---
+
+On OS X 10.10:
+
+1.	If xcode is not already installed, type `gcc` in Terminal; you will be prompted to install the command-line developer tools.  Click 'Install'.
+
+2.	If Python 3 is not already installed, install Python 3.4.3 using the installer from https://www.python.org/downloads/release/python-343/
+
+3.	`pip3 install --user git+https://github.com/ludios/grab-site`
+
+**Important usage note**: Use `~/Library/Python/3.4/bin/` instead of `~/.local/bin/` for all instructions below!
+
+
 Usage
 ---
 

grab-site

@@ -5,7 +5,7 @@ set -e
 url="$1"
 id="$(python3 -c "import os, binascii; print(binascii.hexlify(os.urandom(16)).decode('utf-8'))")"
 # remove protocol, remove trailing slashes, convert slashes to "-"es
-dir="$(echo -n "$url" | sed -r 's,^(ftp|https?)://,,g' | sed -r 's,/+$,,g' | sed -r 's,[/\?&],-,g')-$(date +%F)-${id:0:8}"
+dir="$(echo -nE "$url" | python3 -c "import sys, re; print(re.sub('[/\?&]', '-', sys.stdin.read().strip().split('://', 1)[1].rstrip('/')))")-$(date +%F)-${id:0:8}"
 self=$(dirname "$0")
 
 mkdir -p "$dir"
@@ -45,11 +45,10 @@ for arg in "$@"; do
 	shift
 done
 
-echo
-echo "$id" > "$dir/id"
-echo "$url" > "$dir/start_url"
-echo "$concurrency" > "$dir/concurrency"
-echo "global,$igsets" > "$dir/igsets"
+echo -E "$id" > "$dir/id"
+echo -E "$url" > "$dir/start_url"
+echo -E "$concurrency" > "$dir/concurrency"
+echo -E "global,$igsets" > "$dir/igsets"
 touch "$dir/igoff"
 touch "$dir/ignores"
 

libgrabsite/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.0'
+__version__ = '0.3.0'

libgrabsite/dupes.py

@@ -11,7 +11,11 @@ class DupesOnDisk(object):
 			try:
 				self._env = lmdb.open(
 					filename,
-					writemap=True,
+					# Can't use writemap=True on OS X because it does not fully support sparse files
+					# https://acid.readthedocs.org/en/latest/engines.html
+					# Don't use writemap=True elsewhere because enormous sparse files get copied
+					# and make a non-sparse mess
+					writemap=False,
 					sync=False,
 					metasync=False,
 					# http://lmdb.readthedocs.org/en/release/#lmdb.Environment