401 lines
15 KiB
Bash
401 lines
15 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Fanboy Adblock list grabber script v2.52 (26/03/2016)
|
|
# Dual License CCby3.0/GPLv2
|
|
# http://creativecommons.org/licenses/by/3.0/
|
|
# http://www.gnu.org/licenses/gpl-2.0.html
|
|
#
|
|
# Version history
|
|
#
|
|
# 2.52 Add CookieMonster subscription
|
|
# 2.51 Add Third-party font subscription
|
|
# 2.50 Ultimate/Complete list based on merged list
|
|
# 2.40 Include test Easylist sub generator
|
|
# 2.30 Remove p2p list
|
|
# 2.21 Opera CSS generator
|
|
# 2.20 Allow spaces to prepended to each list before processing
|
|
# 2.15 Optimise sed (no need for temp files)
|
|
# 2.14 More error Checking of Ramdisks, removal of seperate IE ramdisk
|
|
# 2.13 Allow mirrors to use non-ram disk for HG
|
|
# 2.12 Annoyance List split list (beta)
|
|
# 2.11 Added Non-element and Fanboy-Adult
|
|
# 2.10 Tracking List split list
|
|
# 2.06 Better error checking
|
|
# 2.05 Remove Dube loops and create error checking.
|
|
# 2.04 Remove any empty lines
|
|
# 2.03 Allow Hg pulls
|
|
# 2.02 Typo in fanboy-elements-specific.txt
|
|
# 2.01 Various cleanups, add Israeli List
|
|
# 2.00 Re-write the script to support split files
|
|
# 1.82 Better checking of scripts being loaded
|
|
# 1.81 Misc Cleanups
|
|
# 1.8 Allow list to be stored in ramdisk
|
|
# 1.752 Declare global variables
|
|
# 1.751 Remove Shred, and cleanup variable names
|
|
# 1.75 Store log in ramdisk to avoid unnessary writes (Currently disabled)
|
|
# 1.74 Store repo in ramdisk to avoid unnessary writes (Currently disabled)
|
|
#
|
|
# Variables for directorys
|
|
#
|
|
|
|
export ZIP="nice -n 19 /usr/local/bin/7za a -mx=9 -y -tgzip"
|
|
export NICE="nice -n 19"
|
|
export TAC="/usr/bin/tac"
|
|
export CAT="/bin/cat"
|
|
export MAINDIR="/var/www/adblock"
|
|
export SPLITDIR="/var/www/adblock/split/test"
|
|
export HGSERV="/root/fanboy-adblock-list"
|
|
export TESTDIR="/root/tmp/work"
|
|
export DATE="`date`"
|
|
export ADDCHECKSUM="nice -n 19 perl $HGSERV/scripts/addChecksum.pl"
|
|
export LOGFILE="/etc/crons/log.txt"
|
|
export HG="/usr/local/bin/hg"
|
|
export SHA256SUM="/usr/bin/sha256sum"
|
|
export TWIDGE="/usr/bin/twidge update"
|
|
export IEDIR="/var/test/tmp/work/ie"
|
|
export IESUBS="/var/test/ie/subscriptions"
|
|
export EASYLIST="/root/easylist/easylist/easylistfanboy/fanboy-adblock"
|
|
|
|
# Test for Ram disks
|
|
#
|
|
#if [ ! -d "/tmp/work/" ]; then
|
|
# rm -rf /tmp/work/
|
|
# mkdir /tmp/work; chmod 777 /tmp/work
|
|
# mount -t tmpfs -o size=50M tmpfs /tmp/work/
|
|
#fi
|
|
|
|
#if [ ! -d "/tmp/work/opera" ]; then
|
|
# mkdir /tmp/work/opera; chmod 777 /tmp/work/opera
|
|
# mkdir /tmp/work/opera/test; chmod 777 /tmp/work/opera/test
|
|
#fi
|
|
|
|
#if [ ! -d "/tmp/work/opera/test" ]; then
|
|
# mkdir /tmp/work/opera/test; chmod 777 /tmp/work/opera/test
|
|
#fi
|
|
|
|
#if [ ! -d "/tmp/work/split" ]; then
|
|
# mkdir /tmp/work/split; chmod 777 /tmp/work/split
|
|
# mkdir /tmp/work/split/fanboy-adblock; chmod 777 /tmp/work/split/fanboy-adblock
|
|
# mkdir /tmp/work/split/fanboy-addon; chmod 777 /tmp/work/split/fanboy-addon
|
|
# mkdir /tmp/work/split/fanboy-tracking; chmod 777 /tmp/work/split/fanboy-tracking
|
|
#fi
|
|
|
|
#if [ ! -d "/tmp/Ramdisk/" ]; then
|
|
# rm -rf /tmp/Ramdisk/
|
|
# mkdir /tmp/Ramdisk; chmod 777 /tmp/Ramdisk
|
|
# mount -t tmpfs -o size=110M tmpfs /tmp/Ramdisk/
|
|
#fi
|
|
|
|
#if [ ! -d "/tmp/ieramdisk/subscriptions" ]; then
|
|
# rm -rf /tmp/ieramdisk/subscriptions
|
|
# mkdir /tmp/ieramdisk; chmod 777 /tmp/ieramdisk
|
|
# mount -t tmpfs -o size=30M tmpfs /tmp/ieramdisk
|
|
# mkdir /tmp/ieramdisk/subscriptions; chmod 777 /tmp/ieramdisk/subscriptions
|
|
#fi
|
|
|
|
#if [ ! -d "/tmp/Ramdisk/www/adblock/split" ]; then
|
|
# mkdir /tmp/Ramdisk/www/adblock/split; chmod 777 /tmp/Ramdisk/www/adblock/split
|
|
#fi
|
|
|
|
# Check mirror dir exists and its not a symlink
|
|
#
|
|
#if [ -d "/var/hgstuff/fanboy-adblock-list" ] && [ -h "/tmp/hgstuff" ]; then
|
|
# export HGSERV="/var/hgstuff/fanboy-adblock-list"
|
|
# echo "HGSERV=/var/hgstuff/fanboy-adblock-list"
|
|
# cd /tmp/hgstuff/fanboy-adblock-list
|
|
# $NICE $HG pull
|
|
# $NICE $HG update
|
|
# else
|
|
# # If not, its stored here
|
|
# export HGSERV="/tmp/hgstuff/fanboy-adblock-list"
|
|
# echo "HGSERV=/tmp/hgstuff/fanboy-adblock-list"
|
|
# cd /tmp/hgstuff/fanboy-adblock-list
|
|
# $NICE $HG pull
|
|
# $NICE $HG update
|
|
#fi
|
|
|
|
#if [ -d "/root/easylist/easylist" ]; then
|
|
# cd /root/easylist/easylist
|
|
# $NICE $HG pull
|
|
# $NICE $HG update
|
|
#fi
|
|
|
|
############### Fanboy Enhanced Trackers #################
|
|
SSLHG=$($SHA256SUM $HGSERV/enhancedstats-addon.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/enhancedstats.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
# Copy list
|
|
cp -f $HGSERV/enhancedstats-addon.txt $TESTDIR/enhancedstats.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/enhancedstats.txt
|
|
cp -f $TESTDIR/enhancedstats.txt $MAINDIR/enhancedstats.txt
|
|
rm -rf $MAINDIR/enhancedstats.txt.gz
|
|
# GZip
|
|
$ZIP $MAINDIR/enhancedstats.txt.gz $TESTDIR/enhancedstats-addon.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-merged.sh
|
|
# Combine (Main+Tracking+Enhanced) and Ultimate (Main+Tracking+Enhanced+Annoyances)
|
|
$HGSERV/scripts/combine/firefox-adblock-ultimate.sh
|
|
# Firefox2Opera
|
|
# # $NICE $HGSERV/scripts/firefox2opera.sh
|
|
else
|
|
echo "Files are the same: enhancedstats.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy Anti-facebook #################
|
|
SSLHG=$($SHA256SUM $HGSERV/fanboy-antifacebook.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-antifacebook.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
# Copy list
|
|
cp -f $HGSERV/fanboy-antifacebook.txt $TESTDIR/fanboy-antifacebook.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-antifacebook.txt
|
|
cp -f $TESTDIR/fanboy-antifacebook.txt $MAINDIR/fanboy-antifacebook.txt
|
|
rm -rf $MAINDIR/fanboy-antifacebook.txt.gz
|
|
# GZip
|
|
$ZIP $MAINDIR/fanboy-antifacebook.txt.gz $TESTDIR/fanboy-antifacebook.txt > /dev/null
|
|
else
|
|
echo "Files are the same: fanboy-antifacebook.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy CookieMonster #################
|
|
SSLHG=$($SHA256SUM $HGSERV/fanboy-cookiemonster.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-cookiemonster.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
# Copy list
|
|
cp -f $HGSERV/fanboy-cookiemonster.txt $TESTDIR/fanboy-cookiemonster.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-cookiemonster.txt
|
|
cp -f $TESTDIR/fanboy-cookiemonster.txt $MAINDIR/fanboy-cookiemonster.txt
|
|
rm -rf $MAINDIR/fanboy-cookiemonster.txt.gz
|
|
# GZip
|
|
$ZIP $MAINDIR/fanboy-cookiemonster.txt.gz $TESTDIR/fanboy-cookiemonster.txt > /dev/null
|
|
else
|
|
echo "Files are the same: fanboy-cookiemonster.txt" > /dev/null
|
|
fi
|
|
|
|
|
|
|
|
############### Fanboy Anti-Fonts #################
|
|
SSLHG=$($SHA256SUM $HGSERV/fanboy-antifonts.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-antifonts.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
# Copy list
|
|
cp -f $HGSERV/fanboy-antifonts.txt $TESTDIR/fanboy-antifonts.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-antifonts.txt
|
|
cp -f $TESTDIR/fanboy-antifonts.txt $MAINDIR/fanboy-antifonts.txt
|
|
rm -rf $MAINDIR/fanboy-antifonts.txt.gz
|
|
# GZip
|
|
$ZIP $MAINDIR/fanboy-antifonts.txt.gz $TESTDIR/fanboy-antifonts.txt > /dev/null
|
|
else
|
|
echo "Files are the same: fanboy-antifonts.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy CZECH #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-cz.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-czech.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-cz.txt $TESTDIR/fanboy-czech.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-czech.txt
|
|
cp -f $TESTDIR/fanboy-czech.txt $MAINDIR/fanboy-czech.txt
|
|
# Remove old copy, then gzip it
|
|
rm -rf $MAINDIR/fanboy-czech.txt.gz
|
|
$ZIP $MAINDIR/fanboy-czech.txt.gz $TESTDIR/fanboy-czech.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Generate IE script
|
|
# $HGSERV/scripts/ie/czech-ie-generator.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-czech.sh
|
|
else
|
|
echo "Files are the same: fanboy-czech.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy Turkish #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-tky.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-turkish.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-tky.txt $TESTDIR/fanboy-turkish.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-turkish.txt
|
|
cp -f $TESTDIR/fanboy-turkish.txt $MAINDIR/fanboy-turkish.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-turkish.txt.gz
|
|
$ZIP $MAINDIR/fanboy-turkish.txt.gz $TESTDIR/fanboy-turkish.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Generate IE script
|
|
# $HGSERV/scripts/ie/turkish-ie-generator.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-turk.sh
|
|
else
|
|
echo "Files are the same: fanboy-turkish.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy JAPANESE #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-jpn.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-japanese.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-jpn.txt $TESTDIR/fanboy-japanese.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-japanese.txt
|
|
cp -f $TESTDIR/fanboy-japanese.txt $MAINDIR/fanboy-japanese.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-japanese.txt.gz
|
|
$ZIP $MAINDIR/fanboy-japanese.txt.gz $TESTDIR/fanboy-japanese.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Generate IE script
|
|
# $HGSERV/scripts/ie/japanese-ie-generator.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-jpn.sh
|
|
else
|
|
echo "Files are the same: fanboy-japanese.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy KOREAN #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-krn.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-korean.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-krn.txt $TESTDIR/fanboy-korean.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-korean.txt
|
|
cp -f $TESTDIR/fanboy-korean.txt $MAINDIR/fanboy-korean.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-korean.txt.gz
|
|
$ZIP $MAINDIR/fanboy-korean.txt.gz $TESTDIR/fanboy-korean.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-krn.sh
|
|
else
|
|
echo "Files are the same: fanboy-korean.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy POLISH #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-pol.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-polish.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-pol.txt $TESTDIR/fanboy-polish.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-polish.txt
|
|
cp -f $TESTDIR/fanboy-polish.txt $MAINDIR/fanboy-polish.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-polish.txt.gz
|
|
$ZIP $MAINDIR/fanboy-polish.txt.gz $TESTDIR/fanboy-polish.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-pol.sh
|
|
else
|
|
echo "Files are the same: fanboy-polish.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy INDIAN #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-ind.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-indian.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-ind.txt $TESTDIR/fanboy-indian.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-indian.txt
|
|
cp -f $TESTDIR/fanboy-indian.txt $MAINDIR/fanboy-indian.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-indian.txt.gz
|
|
$ZIP $MAINDIR/fanboy-indian.txt.gz $TESTDIR/fanboy-indian.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-ind.sh
|
|
else
|
|
echo "Files are the same: fanboy-indian.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy VIETNAM #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-vtn.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-vietnam.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-vtn.txt $TESTDIR/fanboy-vietnam.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-vietnam.txt
|
|
cp -f $TESTDIR/fanboy-vietnam.txt $MAINDIR/fanboy-vietnam.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-vietnam.txt.gz
|
|
$ZIP $MAINDIR/fanboy-vietnam.txt.gz $TESTDIR/fanboy-vietnam.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-vtn.sh
|
|
else
|
|
echo "Files are the same: fanboy-vietnam.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy ESPANOL #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-esp.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-espanol.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-esp.txt $TESTDIR/fanboy-espanol.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-espanol.txt
|
|
cp -f $TESTDIR/fanboy-espanol.txt $MAINDIR/fanboy-espanol.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-espanol.txt.gz
|
|
$ZIP $MAINDIR/fanboy-espanol.txt.gz $TESTDIR/fanboy-espanol.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Generate IE script
|
|
# $HGSERV/scripts/ie/espanol-ie-generator.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-esp.sh
|
|
else
|
|
echo "Files are the same: fanboy-espanol.txt" > /dev/null
|
|
fi
|
|
|
|
############### Fanboy SWEDISH #################
|
|
SSLHG=$($SHA256SUM $HGSERV/firefox-regional/fanboy-adblocklist-swe.txt | cut -d' ' -f1)
|
|
SSLMAIN=$($SHA256SUM $MAINDIR/fanboy-swedish.txt | cut -d' ' -f1)
|
|
|
|
if [ "$SSLHG" != "$SSLMAIN" ]
|
|
then
|
|
cp -f $HGSERV/firefox-regional/fanboy-adblocklist-swe.txt $TESTDIR/fanboy-swedish.txt
|
|
# Re-generate checksum
|
|
$ADDCHECKSUM $TESTDIR/fanboy-swedish.txt
|
|
cp -f $TESTDIR/fanboy-swedish.txt $MAINDIR/fanboy-swedish.txt
|
|
# Wipe old files
|
|
rm -rf $MAINDIR/fanboy-swedish.txt.gz
|
|
$ZIP $MAINDIR/fanboy-swedish.txt.gz $TESTDIR/fanboy-swedish.txt > /dev/null
|
|
# Combine Regional trackers
|
|
# $HGSERV/scripts/combine/firefox-adblock-intl-tracking.sh
|
|
# Combine
|
|
# $HGSERV/scripts/combine/firefox-adblock-swe.sh
|
|
else
|
|
echo "Files are the same: fanboy-swedish.txt" > /dev/null
|
|
fi
|