magnolia1234
/
bypass-paywalls-chrome-clean
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update custom sites (set referer)

master
magnolia1234 2021-02-06 21:37:40 +01:00
parent e454ff89b4
commit 111864b14c
6 changed files with 68 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -556,7 +556,7 @@ Check 'Options'-link in popup-menu and go to custom sites.
\* by default BPC has limited permissions, but you can opt-in to enable custom sites (and also clear cookies/block general paywall-scripts for non-listed sites).\
Make sure the (new) site is checked under Options (or check on/off-button).
By default sites' cookies/local storage are removed after page loads (to bypass article limit).
Also you can enable Googlebot user-agent or disable Javascript for (sub)domain(s)/external sources.
Also you can enable Googlebot user-agent, disable Javascript for (sub)domain(s)/external sources and/or set the referer (to Facebook, Google or Twitter; ignored when Googlebot is set).
### Add excluded site
Add excluded sites/domains (for your subscriptions).\

58
background.js
View File

@ -204,10 +204,14 @@ var use_bing_bot = [
'themarker.com',
];
var use_facebook_referer = ['clarin.com', 'fd.nl', 'law.com', 'sloanreview.mit.edu'];
var use_twitter_referer = ['medium.com', 'towardsdatascience.com'];
var use_facebook_referer_default = ['clarin.com', 'fd.nl', 'law.com', 'sloanreview.mit.edu'];
var use_facebook_referer = use_facebook_referer_default.slice();
var use_google_referer_default = [];
var use_google_referer = use_google_referer_default.slice();
var use_twitter_referer_default = ['medium.com', 'towardsdatascience.com'];
var use_twitter_referer = use_twitter_referer_default.slice();
var use_random_ip = ['esprit.presse.fr', 'slader.com'];
var change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_twitter_referer, use_random_ip);
var change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
// block paywall-scripts individually
var blockedRegexes = {
@ -494,7 +498,10 @@ function add_grouped_sites(init_rules) {
for (let domain of nl_pg_domains)
remove_cookies_select_drop[domain] = ['TID_ID'];
use_google_bot_default = use_google_bot.slice();
change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_twitter_referer, use_random_ip);
use_facebook_referer_default = use_facebook_referer.slice();
use_google_referer_default = use_google_referer.slice();
use_twitter_referer_default = use_twitter_referer.slice();
change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
}
}
@ -519,6 +526,16 @@ ext_api.storage.local.get({
block_js_custom.push(domainVar);
if (sites_custom[key]['block_javascript_ext'] > 0)
block_js_custom_ext.push(domainVar);
switch (sites_custom[key]['referer']) {
case 'facebook':
use_facebook_referer.push(domainVar);
break;
case 'google':
use_google_referer.push(domainVar);
break;
case 'twitter':
use_twitter_referer.push(domainVar);
}
}
enabledSites = Object.keys(sites).filter(function (key) {
@ -601,6 +618,9 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) {
}
use_google_bot = use_google_bot_default.slice();
use_facebook_referer = use_facebook_referer_default.slice();
use_google_referer = use_google_referer_default.slice();
use_twitter_referer = use_twitter_referer_default.slice();
block_js_custom = [];
block_js_custom_ext = [];
for (let key in sites_custom) {
@ -622,8 +642,18 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) {
if (sites_custom[key]['block_javascript_ext'] > 0) {
block_js_custom_ext.push(domainVar);
}
switch (sites_custom[key]['referer']) {
case 'facebook':
use_facebook_referer.push(domainVar);
break;
case 'google':
use_google_referer.push(domainVar);
break;
case 'twitter':
use_twitter_referer.push(domainVar);
}
}
change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_twitter_referer, use_random_ip);
change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
}
if (key === 'sites_excluded') {
var sites_excluded = storageChange.newValue ? storageChange.newValue : [];
@ -891,12 +921,12 @@ if (['main_frame', 'xmlhttprequest'].includes(details.type) && matchUrlDomain(ch
// if referer exists, set it to google
requestHeaders = requestHeaders.map(function (requestHeader) {
if (requestHeader.name === 'Referer') {
if (matchUrlDomain(use_facebook_referer, details.url)) {
if (matchUrlDomain(use_google_bot, details.url) || matchUrlDomain(use_google_referer, details.url)) {
requestHeader.value = 'https://www.google.com/';
} else if (matchUrlDomain(use_facebook_referer, details.url)) {
requestHeader.value = 'https://www.facebook.com/';
} else if (matchUrlDomain(use_twitter_referer, details.url)) {
requestHeader.value = 'https://t.co/';
} else if (matchUrlDomain(use_google_bot, details.url)) {
requestHeader.value = 'https://www.google.com/';
}
setReferer = true;
}
@ -908,7 +938,12 @@ if (['main_frame', 'xmlhttprequest'].includes(details.type) && matchUrlDomain(ch
// otherwise add it
if (!setReferer) {
if (matchUrlDomain(use_facebook_referer, details.url)) {
if (matchUrlDomain(use_google_bot, details.url) || matchUrlDomain(use_google_referer, details.url)) {
requestHeaders.push({
name: 'Referer',
value: 'https://www.google.com/'
});
} else if (matchUrlDomain(use_facebook_referer, details.url)) {
requestHeaders.push({
name: 'Referer',
value: 'https://www.facebook.com/'
@ -918,11 +953,6 @@ if (['main_frame', 'xmlhttprequest'].includes(details.type) && matchUrlDomain(ch
name: 'Referer',
value: 'https://t.co/'
});
} else {
requestHeaders.push({
name: 'Referer',
value: 'https://www.google.com/'
});
}
}

1
changelog.txt
View File

@ -10,6 +10,7 @@ Fix-update Madsack Mediengruppe (amp)
Fix-update Medium/Towards Data Science (login) & add custom domains in options
Fix-update Stratfor (style; not menu)
Fix-update Sud Ouest (decode text)
Update custom sites (set referer)
* v2.0.6.0 (2021-01-31)
Add Griffith Review (Australia)

2
manifest.json
View File

@ -444,5 +444,5 @@
"*://*.userzoom.com/*",
"*://*.wsj.net/*"
],
"version": "2.0.6.7"
"version": "2.0.6.8"
}

2
options_custom.html
View File

@ -19,7 +19,7 @@
<body>
<h2>Custom Sites</h2>
<div style="width:90%;">
To add a new site, enter an unique title/domain (without www.) and select options for Googlebot/block Javascript (block on (sub)domain(s) of site and/or external domains).<br>
To add a new site, enter an unique title/domain (without www.), select options for Googlebot/block Javascript (block on (sub)domain(s) of site and/or external domains) and/or set referer (ignored when Googlebot is set).<br>
Custom sites are enabled automatically in <small><button><a href="options.html" style="text-decoration:none;color:inherit">Options</a></button></small> (cookies will be removed by default unless you enable allow_cookies).<br>
If you want to use custom sites (for non-listed sites) enable it in <small><button><a href="optin/opt-in.html" style="text-decoration:none;color:inherit">Opt-in</a></button></small><br>
<strong>Custom sites enabled: <span id="custom-enabled"></span></strong>

22
options_custom.js
View File

@ -1,5 +1,7 @@
var ext_api = chrome || browser;
var referer_options = ['', 'facebook', 'google', 'twitter'];
function capitalize(str) {
if (typeof str !== 'string') return '';
return str.charAt(0).toUpperCase() + str.slice(1);
@ -87,7 +89,7 @@ function _imp() {
// Add custom site to ext_api.storage
function add_options() {
var inputEls = document.querySelectorAll('#add_site input');
var inputEls = document.querySelectorAll('#add_site input, #add_site select');
var sites_custom = {};
for (let i = 0; i < inputEls.length; i++) {
@ -178,6 +180,7 @@ function edit_options() {
document.querySelector('input[data-key="allow_cookies"]').checked = (edit_site.allow_cookies > 0);
document.querySelector('input[data-key="block_javascript"]').checked = (edit_site.block_javascript > 0);
document.querySelector('input[data-key="block_javascript_ext"]').checked = (edit_site.block_javascript_ext > 0);
document.querySelector('select[data-key="referer"]').selectedIndex = referer_options.indexOf(edit_site.referer);
});
}
@ -225,6 +228,20 @@ function renderOptions() {
add_sitesEl.appendChild(labelEl);
}
labelEl = document.createElement('label');
labelEl.appendChild(document.createTextNode('referer '));
inputEl = document.createElement('select');
inputEl.dataset.key = 'referer';
labelEl.appendChild(inputEl);
for (var i = 0; i < referer_options.length; i++) {
var option = document.createElement("option");
option.value = referer_options[i];
option.text = referer_options[i];
inputEl.appendChild(option);
}
add_sitesEl.appendChild(labelEl);
// list of custom sites
var custom_sitesEl = document.getElementById('custom_sites');
custom_sitesEl.innerHTML = '';
@ -242,7 +259,8 @@ function renderOptions() {
(sites_custom[key]['googlebot']>0 ? ' | googlebot' : '') +
(sites_custom[key]['allow_cookies']>0 ? ' | allow_cookies' : '') +
(sites_custom[key]['block_javascript']>0 ? ' | block javascript' : '') +
(sites_custom[key]['block_javascript_ext']>0 ? ' | block javascript ext' : '');
(sites_custom[key]['block_javascript_ext']>0 ? ' | block javascript ext' : '') +
(sites_custom[key]['referer'] ? ' | referer: ' + sites_custom[key]['referer'] : '');
optionEl.value = key;
selectEl.add(optionEl);
}