magnolia1234
/
bypass-paywalls-chrome-clean
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update custom sites (set useragent Bingbot)

master
magnolia1234 2021-06-22 21:53:47 +02:00
parent adc4c1a456
commit a548509b37
7 changed files with 69 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -645,7 +645,7 @@ Check 'Options'-link in popup-menu and go to custom sites.
Make sure the (new) site is checked under Options (or check on/off-button).
By default sites' cookies/local storage are removed after page loads (to bypass article limit).
Also you can enable Googlebot user-agent, disable Javascript for (sub)domain(s)/external sources and/or set the referer (to Facebook, Google or Twitter; ignored when Googlebot is set).
Also you can enable Googlebot/Bingbot user-agent, disable Javascript for (sub)domain(s)/external sources and/or set the referer (to Facebook, Google or Twitter; ignored when Googlebot is set).
### Add excluded site
Add excluded sites/domains (for your subscriptions).\

69
background.js
View File

@ -246,12 +246,13 @@ var use_google_bot_default = [
var use_google_bot = use_google_bot_default.slice();
// Override User-Agent with Bingbot
var use_bing_bot = [
var use_bing_bot_default = [
'haaretz.co.il',
'haaretz.com',
'stratfor.com',
'themarker.com',
];
var use_bing_bot = use_bing_bot_default.slice();
var use_facebook_referer_default = ['clarin.com', 'fd.nl', 'ilmanifesto.it', 'law.com', 'sloanreview.mit.edu'];
var use_facebook_referer = use_facebook_referer_default.slice();
@ -576,15 +577,17 @@ function add_grouped_sites(init_rules) {
}
for (let domain of nl_pg_domains)
remove_cookies_select_drop[domain] = ['TID_ID'];
use_google_bot_default = use_google_bot.slice();
use_facebook_referer_default = use_facebook_referer.slice();
use_google_referer_default = use_google_referer.slice();
use_twitter_referer_default = use_twitter_referer.slice();
change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
for (let domain of usa_genomeweb_domains) {
allow_cookies.push(domain);
blockedRegexes[domain] = /crain-platform-.+-prod\.s3\.amazonaws\.com\/s3fs-public\/js\/js_.+\.js/;
}
use_google_bot_default = use_google_bot.slice();
use_bing_bot_default = use_bing_bot.slice();
use_facebook_referer_default = use_facebook_referer.slice();
use_google_referer_default = use_google_referer.slice();
use_twitter_referer_default = use_twitter_referer.slice();
change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
}
}
@ -599,10 +602,37 @@ ext_api.storage.local.get({
var sites_custom = items.sites_custom;
excludedSites = items.sites_excluded;
enabledSites = Object.keys(sites).filter(function (key) {
return (sites[key] !== '' && sites[key] !== '###');
}).map(function (key) {
return sites[key].toLowerCase();
});
customSites = sites_custom;
customSites_domains = Object.values(sites_custom).map(x => x.domain);
disabledSites = defaultSites_domains.concat(customSites_domains).filter(x => !enabledSites.includes(x) && x !== '###');
add_grouped_sites(true); //and exclude sites
for (let domainVar of enabledSites) {
if (!allow_cookies.includes(domainVar) && !remove_cookies.includes(domainVar)) {
allow_cookies.push(domainVar);
remove_cookies.push(domainVar);
}
}
for (let key in sites_custom) {
var domainVar = sites_custom[key]['domain'].toLowerCase();
if (sites_custom[key]['googlebot'] > 0 && !use_google_bot.includes(domainVar))
use_google_bot.push(domainVar);
switch (sites_custom[key]['useragent']) {
case 'googlebot':
if (!use_google_bot.includes(domainVar))
use_google_bot.push(domainVar);
break;
case 'bingbot':
if (!use_bing_bot.includes(domainVar))
use_bing_bot.push(domainVar);
break;
}
if (sites_custom[key]['allow_cookies'] > 0 && !allow_cookies.includes(domainVar))
allow_cookies.push(domainVar);
if (sites_custom[key]['block_javascript'] > 0)
@ -621,22 +651,6 @@ ext_api.storage.local.get({
}
}
enabledSites = Object.keys(sites).filter(function (key) {
return (sites[key] !== '' && sites[key] !== '###');
}).map(function (key) {
return sites[key].toLowerCase();
});
customSites = sites_custom;
customSites_domains = Object.values(sites_custom).map(x => x.domain);
disabledSites = defaultSites_domains.concat(customSites_domains).filter(x => !enabledSites.includes(x) && x !== '###');
add_grouped_sites(true); //and exclude sites
for (let domainVar of enabledSites) {
if (!allow_cookies.includes(domainVar) && !remove_cookies.includes(domainVar)) {
allow_cookies.push(domainVar);
remove_cookies.push(domainVar);
}
}
disableJavascriptOnListedSites();
});
@ -701,6 +715,7 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) {
}
use_google_bot = use_google_bot_default.slice();
use_bing_bot = use_bing_bot_default.slice();
use_facebook_referer = use_facebook_referer_default.slice();
use_google_referer = use_google_referer_default.slice();
use_twitter_referer = use_twitter_referer_default.slice();
@ -711,6 +726,16 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) {
if (sites_custom[key]['googlebot'] > 0 && !use_google_bot.includes(domainVar)) {
use_google_bot.push(domainVar);
}
switch (sites_custom[key]['useragent']) {
case 'googlebot':
if (!use_google_bot.includes(domainVar))
use_google_bot.push(domainVar);
break;
case 'bingbot':
if (!use_bing_bot.includes(domainVar))
use_bing_bot.push(domainVar);
break;
}
if (sites_custom[key]['allow_cookies'] > 0) {
if (allow_cookies.includes(domainVar)) {
if (remove_cookies.includes(domainVar))

1
changelog.txt
View File

@ -3,6 +3,7 @@ Changelog Bypass Paywalls Clean - Chrome
Post-release
Fix blocked referer
Update custom sites (set useragent Bingbot)
* v2.2.6.0 (2021-06-20)
Add group Mediahuis Nederland Regional

2
custom/manifest.json
View File

@ -30,5 +30,5 @@
"webRequestBlocking",
"<all_urls>"
],
"version": "2.2.6.1"
"version": "2.2.6.2"
}

2
manifest.json
View File

@ -517,5 +517,5 @@
"*://*.wallkit.net/*",
"*://*.wsj.net/*"
],
"version": "2.2.6.1"
"version": "2.2.6.2"
}

2
options/options_custom.html
View File

@ -19,7 +19,7 @@
<body>
<h2>Custom Sites</h2>
<div style="width:90%;">
To add a new site, enter an unique title/domain (without www.), select options for Googlebot/block Javascript (block on (sub)domain(s) of site and/or external domains) and/or set referer (ignored when Googlebot is set).<br>
To add a new site, enter an unique title/domain (without www.), select options for useragent (like Googlebot)/block Javascript (block on (sub)domain(s) of site and/or external domains) and/or set referer (ignored when Googlebot is set).<br>
Custom sites are enabled automatically in <small><button><a href="options.html" style="text-decoration:none;color:inherit">Options</a></button></small> (cookies will be removed by default unless you enable allow_cookies).<br>
If you want to use custom sites (for non-listed sites) enable it in <small><button><a href="optin/opt-in.html" style="text-decoration:none;color:inherit">Opt-in</a></button></small>
<strong>Custom sites enabled: <span id="custom-enabled"></span></strong><br>

19
options/options_custom.js
View File

@ -1,5 +1,6 @@
var ext_api = chrome || browser;
var useragent_options = ['', 'googlebot', 'bingbot'];
var referer_options = ['', 'facebook', 'google', 'twitter'];
function capitalize(str) {
@ -176,7 +177,7 @@ function edit_options() {
var edit_site = sites_custom[title];
document.querySelector('input[data-key="title"]').value = title;
document.querySelector('input[data-key="domain"]').value = edit_site.domain;
document.querySelector('input[data-key="googlebot"]').checked = (edit_site.googlebot > 0);
document.querySelector('select[data-key="useragent"]').selectedIndex = (edit_site.googlebot > 0) ? 1 : useragent_options.indexOf(edit_site.useragent);
document.querySelector('input[data-key="allow_cookies"]').checked = (edit_site.allow_cookies > 0);
document.querySelector('input[data-key="block_javascript"]').checked = (edit_site.block_javascript > 0);
document.querySelector('input[data-key="block_javascript_ext"]').checked = (edit_site.block_javascript_ext > 0);
@ -234,7 +235,6 @@ function renderOptions() {
var add_checkboxes = {
'title': 0,
'domain': 0,
'googlebot': 1,
'allow_cookies': 1,
'block_javascript': 1,
'block_javascript_ext': 1
@ -255,6 +255,20 @@ function renderOptions() {
add_sitesEl.appendChild(labelEl);
}
labelEl = document.createElement('label');
labelEl.appendChild(document.createTextNode('useragent '));
inputEl = document.createElement('select');
inputEl.dataset.key = 'useragent';
labelEl.appendChild(inputEl);
for (var i = 0; i < useragent_options.length; i++) {
var option = document.createElement("option");
option.value = useragent_options[i];
option.text = useragent_options[i];
inputEl.appendChild(option);
}
add_sitesEl.appendChild(labelEl);
labelEl = document.createElement('label');
labelEl.appendChild(document.createTextNode('referer '));
inputEl = document.createElement('select');
@ -289,6 +303,7 @@ function renderOptions() {
(sites_custom[key]['allow_cookies']>0 ? ' | allow_cookies' : '') +
(sites_custom[key]['block_javascript']>0 ? ' | block javascript' : '') +
(sites_custom[key]['block_javascript_ext']>0 ? ' | block javascript ext' : '') +
(sites_custom[key]['useragent'] ? ' | useragent: ' + sites_custom[key]['useragent'] : '');
(sites_custom[key]['referer'] ? ' | referer: ' + sites_custom[key]['referer'] : '');
optionEl.value = key;
selectEl.add(optionEl);