#!/bin/bash
# Crawl at a thread rate
threads=4
urls="urls.txt"
output="db.txt"
x=1
while IFS= read -r line
do
echo "Off crawler number ${x}!"
php crawl.php $line $output.$x > webcrawl.log &
((x=x+1))
done < $urls