improve scripts

This commit is contained in:
fanboy 2012-08-23 00:07:17 +12:00
parent 2044603aa8
commit 56109cc5ba
4 changed files with 97 additions and 87 deletions

View File

@ -6,41 +6,43 @@
# http://www.gnu.org/licenses/gpl-2.0.html
#
# Make Ramdisk.
#
$GOOGLEDIR/scripts/ramdisk.sh
# Fallback if ramdisk.sh isn't excuted.
#
if [ ! -d "/tmp/ramdisk/" ]; then
rm -rf /tmp/ramdisk/
mkdir /tmp/ramdisk; chmod 777 /tmp/ramdisk
mount -t tmpfs -o size=30M tmpfs /tmp/ramdisk/
mkdir /tmp/ramdisk/opera/
fi
export ZIP="nice -n 19 /usr/local/bin/7za a -mx=9 -y -tgzip"
export NICE="nice -n 19"
export TAC="/usr/bin/tac"
export CAT="/bin/cat"
export MAINDIR="/tmp/Ramdisk/www/adblock"
export SPLITDIR="/tmp/Ramdisk/www/adblock/split/test"
export HGSERV="/tmp/hgstuff/fanboy-adblock-list"
export TESTDIR="/tmp/work"
export ADDCHECKSUM="nice -n 19 perl $HGSERV/scripts/addChecksum.pl"
export LOGFILE="/etc/crons/log.txt"
export HG="/usr/local/bin/hg"
export SHA256SUM="/usr/bin/sha256sum"
export IEDIR="/tmp/ieramdisk"
export SUBS="/tmp/ieramdisk/subscriptions"
export IRONDIR="/tmp/Ramdisk/www/adblock/iron"
# Copy the Tracking addon from google dir
#
cp -f $GOOGLEDIR/other/tracking-intl.txt $TESTDIR
cp -f $HGSYNC/other/tracking-intl.txt $TESTDIR
sed '1,10d' $GOOGLEDIR/ie/fanboy-tracking-addon.txt > $TESTDIR/fanboy-tracking-addon2.txt
sed '1,10d' $HGSYNC/ie/fanboy-tracking-addon.txt > $TESTDIR/fanboy-tracking-addon2.txt
mv -f $TESTDIR/fanboy-tracking-addon2.txt $TESTDIR/fanboy-tracking-addon.txt
# Copy Filters from the subscriptions..
#
sed -n '/Czech Trackers/,/Slovak Filters/{/Slovak Filters/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-cz.txt > $TESTDIR/fanboy-cz-track.txt
sed -n '/Russian Trackers/,/Russian Whitelist/{/Russian Whitelist/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-rus-v2.txt > $TESTDIR/fanboy-rus-track.txt
sed -n '/Turkish Trackers/,/Firefox 3.x/{/Firefox 3.x/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-tky.txt > $TESTDIR/fanboy-tky-track.txt
sed -n '/Japanese Trackers/,/Japanese Whitelist/{/Japanese Whitelist/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-jpn.txt > $TESTDIR/fanboy-jpn-track.txt
sed -n '/Korean Trackers/,/Korean Specific Whitelists/{/Korean Specific Whitelists/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-krn.txt > $TESTDIR/fanboy-krn-track.txt
sed -n '/Italian Trackers/,/Whitelists/{/Whitelists/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-ita.txt> $TESTDIR/fanboy-ita-track.txt
sed -n '/Polish Trackers/,/Polish Whitelist/{/Polish Whitelist/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-pol.txt > $TESTDIR/fanboy-pol-track.txt
sed -n '/Indian Trackers/,/Indian Whitelists/{/Indian Whitelists/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-ind.txt > $TESTDIR/fanboy-ind-track.txt
sed -n '/Vietnamese Trackers/,/Whitelists/{/Whitelists/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-vtn.txt > $TESTDIR/fanboy-vtn-track.txt
sed -n '/Chinese Trackers/,/Chinese Whitelist/{/Chinese Whitelist/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-chn.txt > $TESTDIR/fanboy-chn-track.txt
sed -n '/Portuguese Trackers/,/Portuguese Generic/{/Portuguese Generic/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-esp.txt > $TESTDIR/fanboy-esp-track.txt
sed -n '/Swedish Trackers/,/Swedish Whitelist/{/Swedish Whitelist/!p}' $GOOGLEDIR/firefox-regional/fanboy-adblocklist-swe.txt > $TESTDIR/fanboy-swe-track.txt
sed -n '/Czech Trackers/,/Slovak Filters/{/Slovak Filters/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-cz.txt > $TESTDIR/fanboy-cz-track.txt
sed -n '/Russian Trackers/,/Russian Whitelist/{/Russian Whitelist/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-rus-v2.txt > $TESTDIR/fanboy-rus-track.txt
sed -n '/Turkish Trackers/,/Firefox 3.x/{/Firefox 3.x/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-tky.txt > $TESTDIR/fanboy-tky-track.txt
sed -n '/Japanese Trackers/,/Japanese Whitelist/{/Japanese Whitelist/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-jpn.txt > $TESTDIR/fanboy-jpn-track.txt
sed -n '/Korean Trackers/,/Korean Specific Whitelists/{/Korean Specific Whitelists/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-krn.txt > $TESTDIR/fanboy-krn-track.txt
sed -n '/Italian Trackers/,/Whitelists/{/Whitelists/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-ita.txt> $TESTDIR/fanboy-ita-track.txt
sed -n '/Polish Trackers/,/Polish Whitelist/{/Polish Whitelist/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-pol.txt > $TESTDIR/fanboy-pol-track.txt
sed -n '/Indian Trackers/,/Indian Whitelists/{/Indian Whitelists/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-ind.txt > $TESTDIR/fanboy-ind-track.txt
sed -n '/Vietnamese Trackers/,/Whitelists/{/Whitelists/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-vtn.txt > $TESTDIR/fanboy-vtn-track.txt
sed -n '/Chinese Trackers/,/Chinese Whitelist/{/Chinese Whitelist/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-chn.txt > $TESTDIR/fanboy-chn-track.txt
sed -n '/Portuguese Trackers/,/Portuguese Generic/{/Portuguese Generic/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-esp.txt > $TESTDIR/fanboy-esp-track.txt
sed -n '/Swedish Trackers/,/Swedish Whitelist/{/Swedish Whitelist/!p}' $HGSYNC/firefox-regional/fanboy-adblocklist-swe.txt > $TESTDIR/fanboy-swe-track.txt
# Remove Dubes and bad filters
#
@ -85,11 +87,11 @@ sed -i '/||waplog.net/d' $TESTDIR/fanboy-rus-track.txt
# Merge for IE trackers
cat $TESTDIR/fanboy-esp-track.txt $TESTDIR/fanboy-cz-track.txt $TESTDIR/fanboy-rus-track.txt $TESTDIR/fanboy-vtn-track.txt $TESTDIR/fanboy-tky-track.txt $TESTDIR/fanboy-jpn-track.txt $TESTDIR/fanboy-krn-track.txt $TESTDIR/fanboy-ita-track.txt $TESTDIR/fanboy-pol-track.txt $TESTDIR/fanboy-chn-track.txt $TESTDIR/fanboy-swe-track.txt $TESTDIR/fanboy-ind-track.txt > $TESTDIR/fanboy-track-test-ie.txt
# Merge enhanced for Firefox
cat $GOOGLEDIR/enhancedstats-addon.txt $TESTDIR/fanboy-track-test-ie.txt > $TESTDIR/fanboy-track-test.txt
cat $HGSYNC/enhancedstats-addon.txt $TESTDIR/fanboy-track-test-ie.txt > $TESTDIR/fanboy-track-test.txt
# Create a backup incase addchecksum "zeros" the file
#
cp -f $TESTDIR/fanboy-track-test.txt $TESTDIR/fanboy-track-bak.txt
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-track-test.txt
$ADDCHECKSUM $TESTDIR/fanboy-track-test.txt
# Now lets check if fanboy-track-test.txt isnt zero
#
@ -102,7 +104,7 @@ then
rm -f $MAINDIR/enhancedstats.txt.gz
# Compress file
#
$NICE $ZIP a -mx=9 -y -tgzip $MAINDIR/enhancedstats.txt.gz $TESTDIR/fanboy-track-test.txt > /dev/null
$ZIP $MAINDIR/enhancedstats.txt.gz $TESTDIR/fanboy-track-test.txt > /dev/null
# Log
echo "Updated enhancedstats.txt (script: firefox-adblock-intl-tracking.sh) on `date +'%Y-%m-%d %H:%M:%S'`" >> /var/log/adblock-log.txt
else
@ -117,7 +119,7 @@ else
rm -f $MAINDIR/enhancedstats.txt.gz
# Compress file
#
$NICE $ZIP a -mx=9 -y -tgzip $MAINDIR/enhancedstats.txt.gz $TESTDIR/fanboy-track-test.txt > /dev/null
$ZIP $MAINDIR/enhancedstats.txt.gz $TESTDIR/fanboy-track-test.txt > /dev/null
# Log
echo "*** ERROR ***: Addchecksum Zero'd the file: enhancedstats.txt (script: firefox-adblock-intl-tracking.sh) on `date +'%Y-%m-%d %H:%M:%S'`" >> /var/log/adblock-log.txt
fi

View File

@ -5,23 +5,27 @@
# http://creativecommons.org/licenses/by/3.0/
# http://www.gnu.org/licenses/gpl-2.0.html
#
# Make Ramdisk.
#
$GOOGLEDIR/scripts/ramdisk.sh
# Fallback if ramdisk.sh isn't excuted.
#
if [ ! -d "/tmp/ramdisk/" ]; then
rm -rf /tmp/ramdisk/
mkdir /tmp/ramdisk; chmod 777 /tmp/ramdisk
mount -t tmpfs -o size=30M tmpfs /tmp/ramdisk/
mkdir /tmp/ramdisk/opera/
fi
export ZIP="nice -n 19 /usr/local/bin/7za a -mx=9 -y -tgzip"
export NICE="nice -n 19"
export TAC="/usr/bin/tac"
export CAT="/bin/cat"
export MAINDIR="/tmp/Ramdisk/www/adblock"
export SPLITDIR="/tmp/Ramdisk/www/adblock/split/test"
export HGSERV="/tmp/hgstuff/fanboy-adblock-list"
export TESTDIR="/tmp/work"
export ADDCHECKSUM="nice -n 19 perl $HGSERV/scripts/addChecksum.pl"
export LOGFILE="/etc/crons/log.txt"
export HG="/usr/local/bin/hg"
export SHA256SUM="/usr/bin/sha256sum"
export IEDIR="/tmp/ieramdisk"
export SUBS="/tmp/ieramdisk/subscriptions"
export IRONDIR="/tmp/Ramdisk/www/adblock/iron"
# Trim off header file (first 2 lines)
#
sed '1,2d' $MAINDIR/enhancedstats.txt > $TESTDIR/fanboy-enhanced.txt
sed '1,2d' $GOOGLEDIR/fanboy-adblocklist-addon.txt > $TESTDIR/fanboy-addon.txt
sed '1,2d' $HGSYNC/fanboy-adblocklist-addon.txt > $TESTDIR/fanboy-addon.txt
sed '1,2d' $MAINDIR/fanboy-tracking-complete.txt > $TESTDIR/fanboy-complete.txt
# Merge to the files together
@ -38,7 +42,7 @@ cp -f $TESTDIR/fanboy-merged.txt $TESTDIR/fanboy-merged-bak.txt
# Add checksum to file
#
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-merged.txt
$ADDCHECKSUM $TESTDIR/fanboy-merged.txt
# Now lets check if fanboy-merged.txt isnt zero
#
@ -53,7 +57,7 @@ then
### echo "Updated fanboy+tracking+addon.txt"
rm -f $MAINDIR/r/fanboy+tracking+addon.txt.gz
# Compress file
$NICE $ZIP a -mx=9 -y -tgzip $MAINDIR/r/fanboy+tracking+addon.txt.gz $MAINDIR/r/fanboy+tracking+addon.txt > /dev/null
$ZIP $MAINDIR/r/fanboy+tracking+addon.txt.gz $MAINDIR/r/fanboy+tracking+addon.txt > /dev/null
# Log
echo "Updated fanboy+tracking+addon.txt (script: firefox-adblock-merged.sh) on `date +'%Y-%m-%d %H:%M:%S'`" >> /var/log/adblock-log.txt
# Clear old Variables
@ -63,7 +67,7 @@ else
# Use the backup file (fanboy-merged.txt was zero'd by addchecksum)
#
sleep 2
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-merged-bak.txt
$ADDCHECKSUM $TESTDIR/fanboy-merged-bak.txt
# Copy Merged file to main dir
#
@ -74,7 +78,7 @@ else
### echo "Updated fanboy+tracking+addon.txt (file was zero)"
rm -f $MAINDIR/r/fanboy+tracking+addon.txt.gz
# Compress file
$NICE $ZIP a -mx=9 -y -tgzip $MAINDIR/r/fanboy+tracking+addon.txt.gz $MAINDIR/r/fanboy+tracking+addon.txt > /dev/null
$ZIP $MAINDIR/r/fanboy+tracking+addon.txt.gz $MAINDIR/r/fanboy+tracking+addon.txt > /dev/null
# Log
echo "*** ERROR ***: Addchecksum Zero'd the file: fanboy-merged.txt (script: firefox-adblock-merged.sh) on `date +'%Y-%m-%d %H:%M:%S'`" >> /var/log/adblock-log.txt
# Clear old Variables

View File

@ -5,24 +5,25 @@
# http://creativecommons.org/licenses/by/3.0/
# http://www.gnu.org/licenses/gpl-2.0.html
#
# Make Ramdisk.
#
$GOOGLEDIR/scripts/ramdisk.sh
# Fallback if ramdisk.sh isn't excuted.
#
if [ ! -d "/tmp/ramdisk/" ]; then
rm -rf /tmp/ramdisk/
mkdir /tmp/ramdisk; chmod 777 /tmp/ramdisk
mount -t tmpfs -o size=30M tmpfs /tmp/ramdisk/
mkdir /tmp/ramdisk/opera/
fi
export ZIP="nice -n 19 /usr/local/bin/7za a -mx=9 -y -tgzip"
export NICE="nice -n 19"
export TAC="/usr/bin/tac"
export CAT="/bin/cat"
export MAINDIR="/tmp/Ramdisk/www/adblock"
export SPLITDIR="/tmp/Ramdisk/www/adblock/split/test"
export HGSERV="/tmp/hgstuff/fanboy-adblock-list"
export TESTDIR="/tmp/work"
export ADDCHECKSUM="nice -n 19 perl $HGSERV/scripts/addChecksum.pl"
export LOGFILE="/etc/crons/log.txt"
export HG="/usr/local/bin/hg"
export SHA256SUM="/usr/bin/sha256sum"
export IEDIR="/tmp/ieramdisk"
export SUBS="/tmp/ieramdisk/subscriptions"
export IRONDIR="/tmp/Ramdisk/www/adblock/iron"
# Trim off header file (first 2 lines)
#
sed '1,2d' $GOOGLEDIR/fanboy-adblocklist-stats.txt > $TESTDIR/fanboy-stats-temp2.txt
sed '1,2d' $HGSYNC/fanboy-adblocklist-stats.txt > $TESTDIR/fanboy-stats-temp2.txt
# Remove Empty Lines
#
@ -35,7 +36,7 @@ sed '$d' < $TESTDIR/fanboy-stats-temp.txt > $TESTDIR/fanboy-stats-temp2.txt
# Merge to the files together
#
cat $MAINDIR/fanboy-adblock.txt $TESTDIR/fanboy-stats-temp2.txt > $TESTDIR/fanboy-stats-merged.txt
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-stats-merged.txt
$ADDCHECKSUM $TESTDIR/fanboy-stats-merged.txt
# Copy Merged file to main dir
#
@ -44,4 +45,4 @@ cp $TESTDIR/fanboy-stats-merged.txt $MAINDIR/r/fanboy+tracking.txt
# Compress file
#
rm -f $MAINDIR/r/fanboy+tracking.txt.gz
$ZIP a -mx=9 -y -tgzip $MAINDIR/r/fanboy+tracking.txt.gz $MAINDIR/r/fanboy+tracking.txt > /dev/null
$ZIP $MAINDIR/r/fanboy+tracking.txt.gz $MAINDIR/r/fanboy+tracking.txt > /dev/null

View File

@ -5,31 +5,34 @@
# http://creativecommons.org/licenses/by/3.0/
# http://www.gnu.org/licenses/gpl-2.0.html
#
# Make Ramdisk.
#
$GOOGLEDIR/scripts/ramdisk.sh
# Fallback if ramdisk.sh isn't excuted.
#
if [ ! -d "/tmp/ramdisk/" ]; then
rm -rf /tmp/ramdisk/
mkdir /tmp/ramdisk; chmod 777 /tmp/ramdisk
mount -t tmpfs -o size=30M tmpfs /tmp/ramdisk/
mkdir /tmp/ramdisk/opera/
fi
export ZIP="nice -n 19 /usr/local/bin/7za a -mx=9 -y -tgzip"
export NICE="nice -n 19"
export TAC="/usr/bin/tac"
export CAT="/bin/cat"
export MAINDIR="/tmp/Ramdisk/www/adblock"
export SPLITDIR="/tmp/Ramdisk/www/adblock/split/test"
export HGSERV="/tmp/hgstuff/fanboy-adblock-list"
export TESTDIR="/tmp/work"
export ADDCHECKSUM="nice -n 19 perl $HGSERV/scripts/addChecksum.pl"
export LOGFILE="/etc/crons/log.txt"
export HG="/usr/local/bin/hg"
export SHA256SUM="/usr/bin/sha256sum"
export IEDIR="/tmp/ieramdisk"
export SUBS="/tmp/ieramdisk/subscriptions"
export IRONDIR="/tmp/Ramdisk/www/adblock/iron"
# Clear old files
#
rm -rf $TESTDIR/fanboy-addon-temp*.txt $TESTDIR/enhancedstats-addon-temp*.txt $TESTDIR/fanboy-stats-temp*.txt $TESTDIR/fanboy-complete.txt $TESTDIR/fanboy-ultimate.txt
# Tracking filter: Trim off header file, remove empty lines, and bottom line
sed '1,9d' $GOOGLEDIR/fanboy-adblocklist-stats.txt > $TESTDIR/fanboy-stats-temp2.txt
sed '1,9d' $HGSYNC/fanboy-adblocklist-stats.txt > $TESTDIR/fanboy-stats-temp2.txt
sed '/^$/d' $TESTDIR/fanboy-stats-temp2.txt > $TESTDIR/fanboy-stats-temp3.txt
sed '$d' < $TESTDIR/fanboy-stats-temp3.txt > $TESTDIR/fanboy-stats-temp.txt
# Annoyances filter: Trim off header file, remove empty lines, and bottom line
sed '1,10d' $GOOGLEDIR/fanboy-adblocklist-addon.txt > $TESTDIR/fanboy-addon-temp2.txt
sed '1,10d' $HGSYNC/fanboy-adblocklist-addon.txt > $TESTDIR/fanboy-addon-temp2.txt
sed '/^$/d' $TESTDIR/fanboy-addon-temp2.txt > $TESTDIR/fanboy-addon-temp3.txt
# Enhanced-tracking filter: Trim off header file, remove empty lines, and bottom line
@ -44,7 +47,7 @@ sed -i '/\/js\/tracking.js/d' $TESTDIR/fanboy-stats-temp.txt
# Insert a new line to avoid chars running into each other
#
cat $GOOGLEDIR/fanboy-adblocklist-current-expanded.txt | sed '$a!' > $TESTDIR/fanboy-adblocklist-current.txt
cat $HGSYNC/fanboy-adblocklist-current-expanded.txt | sed '$a!' > $TESTDIR/fanboy-adblocklist-current.txt
# Merge to the files together
#
@ -69,8 +72,8 @@ cp -f $TESTDIR/fanboy-ultimate.txt $TESTDIR/fanboy-ultimate-bak.txt
# Addchecksum
#
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-complete.txt
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-ultimate.txt
$ADDCHECKSUM $TESTDIR/fanboy-complete.txt
$ADDCHECKSUM $TESTDIR/fanboy-ultimate.txt
# Now lets check if fanboy-merged.txt isnt zero
@ -88,8 +91,8 @@ then
# Compress Files
#
$NICE $ZIP a -mx=9 -y -tgzip $TESTDIR/fanboy-complete.txt.gz $TESTDIR/fanboy-complete.txt > /dev/null
$NICE $ZIP a -mx=9 -y -tgzip $TESTDIR/fanboy-ultimate.txt.gz $TESTDIR/fanboy-ultimate.txt > /dev/null
$ZIP $TESTDIR/fanboy-complete.txt.gz $TESTDIR/fanboy-complete.txt > /dev/null
$ZIP $TESTDIR/fanboy-ultimate.txt.gz $TESTDIR/fanboy-ultimate.txt > /dev/null
# Copy to server
#
@ -130,9 +133,9 @@ else
sleep 2
# Addchecksum
#
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-complete-bak.txt
$ADDCHECKSUM $TESTDIR/fanboy-complete-bak.txt
sleep 2
perl $TESTDIR/addChecksum.pl $TESTDIR/fanboy-ultimate-bak.txt
$ADDCHECKSUM $TESTDIR/fanboy-ultimate-bak.txt
# Copy Merged file to main dir
#
@ -145,9 +148,9 @@ else
# Compress Files
#
$NICE $ZIP a -mx=9 -y -tgzip $TESTDIR/fanboy-complete.txt.gz $TESTDIR/fanboy-complete-bak.txt > /dev/null
$ZIP $TESTDIR/fanboy-complete.txt.gz $TESTDIR/fanboy-complete-bak.txt > /dev/null
sleep 2
$NICE $ZIP a -mx=9 -y -tgzip $TESTDIR/fanboy-ultimate.txt.gz $TESTDIR/fanboy-ultimate-bak.txt > /dev/null
$ZIP $TESTDIR/fanboy-ultimate.txt.gz $TESTDIR/fanboy-ultimate-bak.txt > /dev/null
# Copy to server
#