diff --git a/README.md b/README.md index c44ee46..e3ecf65 100755 --- a/README.md +++ b/README.md @@ -556,7 +556,7 @@ Check 'Options'-link in popup-menu and go to custom sites. \* by default BPC has limited permissions, but you can opt-in to enable custom sites (and also clear cookies/block general paywall-scripts for non-listed sites).\ Make sure the (new) site is checked under Options (or check on/off-button). By default sites' cookies/local storage are removed after page loads (to bypass article limit). -Also you can enable Googlebot user-agent or disable Javascript for (sub)domain(s)/external sources. +Also you can enable Googlebot user-agent, disable Javascript for (sub)domain(s)/external sources and/or set the referer (to Facebook, Google or Twitter; ignored when Googlebot is set). ### Add excluded site Add excluded sites/domains (for your subscriptions).\ diff --git a/background.js b/background.js index 005b02b..b23f52b 100755 --- a/background.js +++ b/background.js @@ -204,10 +204,14 @@ var use_bing_bot = [ 'themarker.com', ]; -var use_facebook_referer = ['clarin.com', 'fd.nl', 'law.com', 'sloanreview.mit.edu']; -var use_twitter_referer = ['medium.com', 'towardsdatascience.com']; +var use_facebook_referer_default = ['clarin.com', 'fd.nl', 'law.com', 'sloanreview.mit.edu']; +var use_facebook_referer = use_facebook_referer_default.slice(); +var use_google_referer_default = []; +var use_google_referer = use_google_referer_default.slice(); +var use_twitter_referer_default = ['medium.com', 'towardsdatascience.com']; +var use_twitter_referer = use_twitter_referer_default.slice(); var use_random_ip = ['esprit.presse.fr', 'slader.com']; -var change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_twitter_referer, use_random_ip); +var change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip); // block paywall-scripts individually var blockedRegexes = { @@ -494,7 +498,10 @@ function add_grouped_sites(init_rules) { for (let domain of nl_pg_domains) remove_cookies_select_drop[domain] = ['TID_ID']; use_google_bot_default = use_google_bot.slice(); - change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_twitter_referer, use_random_ip); + use_facebook_referer_default = use_facebook_referer.slice(); + use_google_referer_default = use_google_referer.slice(); + use_twitter_referer_default = use_twitter_referer.slice(); + change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip); } } @@ -519,6 +526,16 @@ ext_api.storage.local.get({ block_js_custom.push(domainVar); if (sites_custom[key]['block_javascript_ext'] > 0) block_js_custom_ext.push(domainVar); + switch (sites_custom[key]['referer']) { + case 'facebook': + use_facebook_referer.push(domainVar); + break; + case 'google': + use_google_referer.push(domainVar); + break; + case 'twitter': + use_twitter_referer.push(domainVar); + } } enabledSites = Object.keys(sites).filter(function (key) { @@ -601,6 +618,9 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) { } use_google_bot = use_google_bot_default.slice(); + use_facebook_referer = use_facebook_referer_default.slice(); + use_google_referer = use_google_referer_default.slice(); + use_twitter_referer = use_twitter_referer_default.slice(); block_js_custom = []; block_js_custom_ext = []; for (let key in sites_custom) { @@ -622,8 +642,18 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) { if (sites_custom[key]['block_javascript_ext'] > 0) { block_js_custom_ext.push(domainVar); } + switch (sites_custom[key]['referer']) { + case 'facebook': + use_facebook_referer.push(domainVar); + break; + case 'google': + use_google_referer.push(domainVar); + break; + case 'twitter': + use_twitter_referer.push(domainVar); + } } - change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_twitter_referer, use_random_ip); + change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip); } if (key === 'sites_excluded') { var sites_excluded = storageChange.newValue ? storageChange.newValue : []; @@ -891,12 +921,12 @@ if (['main_frame', 'xmlhttprequest'].includes(details.type) && matchUrlDomain(ch // if referer exists, set it to google requestHeaders = requestHeaders.map(function (requestHeader) { if (requestHeader.name === 'Referer') { - if (matchUrlDomain(use_facebook_referer, details.url)) { + if (matchUrlDomain(use_google_bot, details.url) || matchUrlDomain(use_google_referer, details.url)) { + requestHeader.value = 'https://www.google.com/'; + } else if (matchUrlDomain(use_facebook_referer, details.url)) { requestHeader.value = 'https://www.facebook.com/'; } else if (matchUrlDomain(use_twitter_referer, details.url)) { requestHeader.value = 'https://t.co/'; - } else if (matchUrlDomain(use_google_bot, details.url)) { - requestHeader.value = 'https://www.google.com/'; } setReferer = true; } @@ -908,7 +938,12 @@ if (['main_frame', 'xmlhttprequest'].includes(details.type) && matchUrlDomain(ch // otherwise add it if (!setReferer) { - if (matchUrlDomain(use_facebook_referer, details.url)) { + if (matchUrlDomain(use_google_bot, details.url) || matchUrlDomain(use_google_referer, details.url)) { + requestHeaders.push({ + name: 'Referer', + value: 'https://www.google.com/' + }); + } else if (matchUrlDomain(use_facebook_referer, details.url)) { requestHeaders.push({ name: 'Referer', value: 'https://www.facebook.com/' @@ -918,11 +953,6 @@ if (['main_frame', 'xmlhttprequest'].includes(details.type) && matchUrlDomain(ch name: 'Referer', value: 'https://t.co/' }); - } else { - requestHeaders.push({ - name: 'Referer', - value: 'https://www.google.com/' - }); } } diff --git a/changelog.txt b/changelog.txt index 92b1722..cb34507 100644 --- a/changelog.txt +++ b/changelog.txt @@ -10,6 +10,7 @@ Fix-update Madsack Mediengruppe (amp) Fix-update Medium/Towards Data Science (login) & add custom domains in options Fix-update Stratfor (style; not menu) Fix-update Sud Ouest (decode text) +Update custom sites (set referer) * v2.0.6.0 (2021-01-31) Add Griffith Review (Australia) diff --git a/manifest.json b/manifest.json index 2689844..2efa411 100755 --- a/manifest.json +++ b/manifest.json @@ -444,5 +444,5 @@ "*://*.userzoom.com/*", "*://*.wsj.net/*" ], - "version": "2.0.6.7" + "version": "2.0.6.8" } \ No newline at end of file diff --git a/options_custom.html b/options_custom.html index f2c2a5e..b7b804b 100644 --- a/options_custom.html +++ b/options_custom.html @@ -19,7 +19,7 @@

Custom Sites

- To add a new site, enter an unique title/domain (without www.) and select options for Googlebot/block Javascript (block on (sub)domain(s) of site and/or external domains).
+ To add a new site, enter an unique title/domain (without www.), select options for Googlebot/block Javascript (block on (sub)domain(s) of site and/or external domains) and/or set referer (ignored when Googlebot is set).
Custom sites are enabled automatically in (cookies will be removed by default unless you enable allow_cookies).
If you want to use custom sites (for non-listed sites) enable it in
Custom sites enabled: diff --git a/options_custom.js b/options_custom.js index fcae036..1ed1605 100644 --- a/options_custom.js +++ b/options_custom.js @@ -1,5 +1,7 @@ var ext_api = chrome || browser; +var referer_options = ['', 'facebook', 'google', 'twitter']; + function capitalize(str) { if (typeof str !== 'string') return ''; return str.charAt(0).toUpperCase() + str.slice(1); @@ -87,7 +89,7 @@ function _imp() { // Add custom site to ext_api.storage function add_options() { - var inputEls = document.querySelectorAll('#add_site input'); + var inputEls = document.querySelectorAll('#add_site input, #add_site select'); var sites_custom = {}; for (let i = 0; i < inputEls.length; i++) { @@ -178,6 +180,7 @@ function edit_options() { document.querySelector('input[data-key="allow_cookies"]').checked = (edit_site.allow_cookies > 0); document.querySelector('input[data-key="block_javascript"]').checked = (edit_site.block_javascript > 0); document.querySelector('input[data-key="block_javascript_ext"]').checked = (edit_site.block_javascript_ext > 0); + document.querySelector('select[data-key="referer"]').selectedIndex = referer_options.indexOf(edit_site.referer); }); } @@ -225,6 +228,20 @@ function renderOptions() { add_sitesEl.appendChild(labelEl); } + labelEl = document.createElement('label'); + labelEl.appendChild(document.createTextNode('referer ')); + inputEl = document.createElement('select'); + inputEl.dataset.key = 'referer'; + labelEl.appendChild(inputEl); + + for (var i = 0; i < referer_options.length; i++) { + var option = document.createElement("option"); + option.value = referer_options[i]; + option.text = referer_options[i]; + inputEl.appendChild(option); + } + add_sitesEl.appendChild(labelEl); + // list of custom sites var custom_sitesEl = document.getElementById('custom_sites'); custom_sitesEl.innerHTML = ''; @@ -242,7 +259,8 @@ function renderOptions() { (sites_custom[key]['googlebot']>0 ? ' | googlebot' : '') + (sites_custom[key]['allow_cookies']>0 ? ' | allow_cookies' : '') + (sites_custom[key]['block_javascript']>0 ? ' | block javascript' : '') + - (sites_custom[key]['block_javascript_ext']>0 ? ' | block javascript ext' : ''); + (sites_custom[key]['block_javascript_ext']>0 ? ' | block javascript ext' : '') + + (sites_custom[key]['referer'] ? ' | referer: ' + sites_custom[key]['referer'] : ''); optionEl.value = key; selectEl.add(optionEl); }