Add Substack custom domains (opt-in to custom sites)

2022-08-17 08:48:54 +02:00 · 2022-08-17 08:48:54 +02:00 · c1dc8ae307
parent 82af78b108
commit c1dc8ae307
7 changed files with 48 additions and 62 deletions
--- a/README.md
+++ b/README.md
@ -171,11 +171,13 @@ Grouped in options:\
 [StuDocu](https://www.studocu.com)

 ##### Magazines/Blogs
-[Medium](https://www.medium.com)*\
+[Medium](https://www.medium.com)* -
+[Substack](https://substack.com)\
 Grouped in options:\
 *Medium custom domains* like (opt-in to custom sites for unlisted)\
 [Better Programming](https://betterprogramming.pub) -
-[Towards Data Science](https://towardsdatascience.com)
+[Towards Data Science](https://towardsdatascience.com)\
+*Substack custom domains* (opt-in to custom sites)

 [Apollo Magazine](https://www.apollo-magazine.com) -
 [Artforum](https://www.artforum.com) -
@ -190,7 +192,6 @@ Grouped in options:\
 [Slate](https://slate.com) -
 [SofRep](https://sofrep.com) -
 [Sports Illustrated](https://www.si.com) -
-[Substack](https://substack.com) -
 [The Art Newspaper](https://www.theartnewspaper.com) -
 [The Athletic](https://theathletic.com) -
 [The Intercept](https://theintercept.com) -
--- a/background.js
+++ b/background.js
@ -97,6 +97,7 @@ function initSetRules() {
  block_js_custom = [];
  block_js_custom_ext = [];
  blockedRegexes = {};
+  init_custom_domains();
 }

 const userAgentDesktopG = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
@ -759,7 +760,7 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
        break;
      }
    }
-    var blocked_referer_domains = ['foreignaffairs.com', 'timeshighereducation.com'];
+    var blocked_referer_domains = ['timeshighereducation.com'];
    if (!header_referer && details.initiator) {
      header_referer = details.initiator;
      if (!blocked_referer && matchUrlDomain(blocked_referer_domains, details.url) && ['script', 'xmlhttprequest'].includes(details.type)) {
@ -807,12 +808,17 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
    return custom_domains;
  }

-  if (!matchUrlDomain(excludedSites, header_referer)) {
+  if (!matchUrlDomain(enabledSites.concat(disabledSites, excludedSites), header_referer)) {

  // remove cookies for sites medium platform (custom domains)
  var medium_custom_domain = (matchUrlDomain('cdn-client.medium.com', details.url) && ['script'].includes(details.type) && !matchUrlDomain(medium_custom_domains.concat(['medium.com']), header_referer) && enabledSites.includes('###_medium_custom'));
  if (medium_custom_domain)
    medium_custom_domains = customAddRules(medium_custom_domains);
+  else {
+    // remove cookies & set googlebot-useragent for sites substack platform (custom domains)
+    var substack_custom_domain = (matchUrlDomain('substack.com', details.url) && ['sub_frame', 'other'].includes(details.type) && !matchUrlDomain(substack_custom_domains.concat(['substack.com']), header_referer) && enabledSites.includes('###_substack_custom'));
+    if (substack_custom_domain)
+      substack_custom_domains = customAddRules(substack_custom_domains, '', '', 'googlebot');
  else {
    let header_referer_hostname = urlHost(header_referer);
    if (header_referer_hostname.match(/\.(com|net)\.au$/)) {
@ -822,7 +828,6 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
        au_comm_media_domains = customAddRules(au_comm_media_domains, {allow_cookies: 1}, blockedRegexes['canberratimes.com.au']);
      else if (header_referer_hostname.endsWith('.com.au')) {
        // enable regional The West Australian sites (opt-in to custom sites)
-        var au_thewest_domains = ['thewest.com.au'];
        var au_thewest_domain = (details.url.startsWith('https://images.thewest.com.au/') && ['image'].includes(details.type) &&
          !matchUrlDomain(au_thewest_domains, header_referer) && enabledSites.includes('thewest.com.au'));
        if (au_thewest_domain)
@ -830,47 +835,38 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
      }
    } else if (header_referer_hostname.endsWith('.ch')) {
      // set googlebot-useragent for regional nzz.ch sites (opt-in to custom sites)
-      var ch_media_domains = [];
      var ch_media_domain = (matchUrlDomain('static-chmedia.ch', details.url) && ['script'].includes(details.type) && !matchUrlDomain(ch_media_domains, header_referer) && enabledSites.includes('nzz.ch'));
      if (ch_media_domain)
        ch_media_domains = customAddRules(ch_media_domains, {allow_cookies: 1}, blockedRegexes['nzz.ch'], 'googlebot');
    } else if (header_referer_hostname.endsWith('.cl')) {
      // block scripts for regional El Mercurio sites (opt-in to custom sites)
-      var cl_emol_region_domains = [];
      var cl_emol_region_domain = (matchUrlDomain('impresa.soy-chile.cl', details.url) && ['xmlhttprequest'].includes(details.type) && !matchUrlDomain(cl_emol_region_domains, header_referer) && enabledSites.includes('elmercurio.com'));
      if (cl_emol_region_domain) {
        cl_emol_region_domains = customAddRules(cl_emol_region_domains, '', "(\\.{domain}\\/impresa\\/.+\\/assets\\/(vendor|\\d)\\.js|pram\\.pasedigital\\.cl\\/API\\/User\\/Status\\?)");
      }
    } else if (header_referer_hostname.endsWith('.de')) {
      // set googlebot-useragent for additional Funke sites (opt-in to custom sites)
-      var de_funke_medien_domains = grouped_sites['###_de_funke_medien'];
      var de_funke_domain = (matchUrlDomain('funkedigital.de', details.url) && ['script'].includes(details.type) && !matchUrlDomain(de_funke_medien_domains, header_referer) && enabledSites.includes('###_de_funke_medien'));
      if (de_funke_domain)
        de_funke_medien_domains = customAddRules(de_funke_medien_domains, {allow_cookies: 1}, blockedRegexes['waz.de'], 'googlebot');
      else {
        // block script for additional Madsack/RND sites (opt-in to custom sites)
-        var de_madsack_domains = grouped_sites['###_de_madsack'];
-        var de_madsack_custom_domains = ['aller-zeitung.de', 'dnn.de', 'gnz.de', 'goettinger-tageblatt.de', 'paz-online.de', 'sn-online.de', 'waz-online.de'];
        var de_madsack_domain = (matchUrlDomain(de_madsack_custom_domains, details.url) && !matchUrlDomain(de_madsack_domains, header_referer) && enabledSites.includes('###_de_madsack'));
        if (de_madsack_domain)
          de_madsack_domains = customAddRules(de_madsack_domains, {allow_cookies: 1}, blockedRegexes['haz.de']);
      }
    } else if (header_referer_hostname.match(/\.(es|cat)$/) || matchUrlDomain(['diariocordoba.com', 'elperiodicodearagon.com', 'elperiodicoextremadura.com', 'elperiodicomediterraneo.com', 'emporda.info'], header_referer)) {
      // block Piano.io for unlisted Grupo Prensa Ibérica (opt-in to custom sites)
-      var es_epiberica_domains = grouped_sites['###_es_epiberica'];
-      var es_epiberica_custom_domains = ['diaridegirona.cat', 'diariocordoba.com', 'diariodeibiza.es', 'elperiodicodearagon.com', 'elperiodicoextremadura.com', 'elperiodicomediterraneo.com', 'emporda.info', 'laopinioncoruna.es', 'laopiniondemalaga.es', 'laopiniondemurcia.es', 'laopiniondezamora.es', 'regio7.cat'];
      var es_epiberica_domain = (matchUrlDomain(es_epiberica_custom_domains, details.url) && !matchUrlDomain(es_epiberica_domains, header_referer) && enabledSites.includes('###_es_epiberica'));
      if (es_epiberica_domain)
        es_epiberica_domains = customAddRules(es_epiberica_domains, {allow_cookies: 1}, blockedRegexes['epe.es']);
    } else if (header_referer_hostname.endsWith('.fi')) {
      // set user-agent to GoogleBot for additional Snamoma Media Finland (opt-in to custom sites)
-      var fi_sanoma_domains = grouped_sites['###_fi_sanoma'];
      var fi_sanoma_sndp_domain = (matchUrlDomain('sanoma-sndp.fi', details.url) && ['xmlhttprequest'].includes(details.type) && !matchUrlDomain(fi_sanoma_domains, header_referer) && enabledSites.includes('###_fi_sanoma'));
      if (fi_sanoma_sndp_domain)
        fi_sanoma_domains = customAddRules(fi_sanoma_domains, {allow_cookies: 1}, '', 'googlebot');
    } else if (header_referer_hostname.endsWith('.nl')) {
      // block Evolok for Mediahuis Noord sites (opt-in to custom sites)
-      var nl_mediahuis_noord_domains = [];
      var nl_mediahuis_noord_domain = (matchUrlDomain('ndcmediagroep.nl', details.url) && ['script'].includes(details.type) && !matchUrlDomain(nl_mediahuis_noord_domains, header_referer) && enabledSites.includes('###_nl_mediahuis_noord'));
      if (nl_mediahuis_noord_domain)
        nl_mediahuis_noord_domains = customAddRules(nl_mediahuis_noord_domains, {allow_cookies: 1}, blockedRegexes['lc.nl']);
@ -881,38 +877,32 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
        ca_gcm_domains = customAddRules(ca_gcm_domains, {allow_cookies: 1, remove_cookies: 1});
      else {
        // block TinyPass for Postmedia Network sites
-        var ca_postmedia_domains = grouped_sites['###_ca_postmedia'];
        var ca_postmedia_domain = (matchUrlDomain('postmedia.digital', details.url) && ['script'].includes(details.type) && !matchUrlDomain(ca_postmedia_domains.concat(['canada.com', 'canoe.com', 'driving.ca']), header_referer) && enabledSites.includes('###_ca_postmedia'));
        if (ca_postmedia_domain)
          ca_postmedia_domains = customAddRules(ca_postmedia_domains, {allow_cookies: 1}, blockedRegexes['nationalpost.com']);
        else {
          // set googlebot-useragent for Gannett sites
-          var usa_gannett_domains = grouped_sites['###_usa_gannett'];
          var usa_gannett_domain = (matchUrlDomain('gannett-cdn.com', details.url) && ['xmlhttprequest'].includes(details.type) && !matchUrlDomain(usa_gannett_domains.concat(['usatoday.com']), header_referer) && enabledSites.includes('###_usa_gannett'));
          if (usa_gannett_domain)
            usa_gannett_domains = customAddRules(usa_gannett_domains, '', blockedRegexes['azcentral.com'], 'googlebot', '', true);
          else {
-            var usa_hearst_comm_domains = grouped_sites['###_usa_hearst_comm'];
            var usa_hearst_comm_domain = (matchUrlDomain('treg.hearstnp.com', details.url) && ['script'].includes(details.type) && !matchUrlDomain(usa_hearst_comm_domains, header_referer) && enabledSites.includes('###_usa_hearst_comm'));
            if (usa_hearst_comm_domain)
              usa_hearst_comm_domains = customAddRules(usa_hearst_comm_domains, '', blockedRegexes['houstonchronicle.com']);
            else {
              // block script for additional Lee Enterprises sites (opt-in to custom sites)
-              var usa_lee_ent_domains = grouped_sites['###_usa_lee_ent'];
              var usa_lee_ent_domain = (details.url.match(/\.townnews\.com\/(central\.)?leetemplates\.com\//) && ['script'].includes(details.type) &&
                !matchUrlDomain(usa_lee_ent_domains, header_referer) && enabledSites.includes('###_usa_lee_ent'));
              if (usa_lee_ent_domain)
                usa_lee_ent_domains = customAddRules(usa_lee_ent_domains, '', blockedRegexes['buffalonews.com']);
              else {
                // block script for TownNews sites (Blox CMS; opt-in to custom sites)
-                var usa_townnews_domains = [];
                var usa_townnews_domain = (details.url.match(/\.townnews\.com\/.+\/tncms\//) && ['script'].includes(details.type) &&
                  !matchUrlDomain(usa_townnews_domains.concat(usa_lee_ent_domains, ['townnews.com', 'galvnews.com']), header_referer) && enabledSites.includes('###_usa_townnews'));
                if (usa_townnews_domain)
                  usa_townnews_domains = customAddRules(usa_townnews_domains, '', /\.com\/shared-content\/art\/tncms\/user\/user\.js/);
                else {
                  // block script for additional McClatchy sites (opt-in to custom sites)
-                  var usa_mcc_domains = grouped_sites['###_usa_mcc'];
                  var usa_mcc_domain = (((matchUrlDomain('mcclatchyinteractive.com', details.url) && ['script'].includes(details.type)) ||
                      (matchUrlDomain('mcclatchy.com', details.url) && ['image'].includes(details.type))) &&
                    !matchUrlDomain(usa_mcc_domains.concat(['mcclatchy.com']), header_referer) && enabledSites.includes('###_usa_mcc'));
@ -920,7 +910,6 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
                    usa_mcc_domains = customAddRules(usa_mcc_domains, '', blockedRegexes['bnd.com']);
                  else {
                    // block script for additional MediaNews Group sites (opt-in to custom sites)
-                    var usa_mng_domains = grouped_sites['###_usa_mng'];
                    var usa_mng_domain = (details.url.match(/\.com\/wp-content\/plugins\/dfm(-|_).+\.js/) && ['script'].includes(details.type) &&
                      !matchUrlDomain(usa_mng_domains, header_referer) && enabledSites.includes('###_usa_mng'));
                    if (usa_mng_domain)
@ -932,7 +921,8 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
          }
        }
      }
-    }
+  }
+  }
  }

  } // not in excludedSites
--- a/changelog.txt
+++ b/changelog.txt
@ -2,6 +2,7 @@
 Changelog Bypass Paywalls Clean - Chrome

 Post-release
+Add Substack custom domains (opt-in to custom sites)
 Fix El Correo & La Voz de Cadiz

 * v2.8.0.0 (2022-08-15)
--- a/custom/manifest.json
+++ b/custom/manifest.json
@ -43,5 +43,5 @@
        "webRequestBlocking",
        "<all_urls>"
    ],
-    "version": "2.8.0.1"
+    "version": "2.8.0.2"
 }
--- a/manifest.json
+++ b/manifest.json
@ -687,5 +687,5 @@
    "*://gcm.omerlocdn.com/*",
    "*://webcache.googleusercontent.com/*"
  ],
-  "version": "2.8.0.1"
+  "version": "2.8.0.2"
 }
--- a/sites.js
+++ b/sites.js
@ -1569,6 +1569,9 @@ var defaultSites = {
    domain: "substack.com",
    useragent: "googlebot"
  },
+  "Substack custom domains (opt-in to custom sites)": {
+    domain: "###_substack_custom"
+  },
  "Suomen Sotilas": {
    domain: "suomensotilas.fi",
    allow_cookies: 1,
@ -2132,17 +2135,42 @@ var grouped_sites = {};
 expandSiteRules(defaultSites);

 // grouped domains (background)
-var au_comm_media_domains = grouped_sites['###_au_comm_media'];
 var au_news_corp_domains = grouped_sites['###_au_news_corp'];
 var be_mediahuis_domains = grouped_sites['###_be_mediahuis'];
-var ca_gcm_domains = grouped_sites['###_ca_gcm'];
 var de_westfalen_medien_domains = grouped_sites['###_de_westfalen_medien'];
 var fr_groupe_sud_ouest_domains = grouped_sites['###_fr_groupe_sud_ouest'];
 var it_gedi_domains = grouped_sites['###_it_gedi'];
-var medium_custom_domains = grouped_sites['###_medium_custom'];
 var nl_mediahuis_region_domains = grouped_sites['###_nl_mediahuis_region'];
 var no_nhst_media_domains = grouped_sites['###_no_nhst_media'];

+// (fixed) custom domains (background)
+var au_comm_media_domains, au_thewest_domains, ca_gcm_domains, ca_postmedia_domains, ch_media_domains, cl_emol_region_domains, de_funke_medien_domains, de_madsack_domains, es_epiberica_domains, fi_sanoma_domains, medium_custom_domains, nl_mediahuis_noord_domains, substack_custom_domains, usa_gannett_domains, usa_hearst_comm_domains, usa_lee_ent_domains, usa_mcc_domains, usa_mng_domains, usa_townnews_domains;
+var de_madsack_custom_domains = ['aller-zeitung.de', 'dnn.de', 'gnz.de', 'goettinger-tageblatt.de', 'paz-online.de', 'sn-online.de', 'waz-online.de'];
+var es_epiberica_custom_domains = ['diaridegirona.cat', 'diariocordoba.com', 'diariodeibiza.es', 'elperiodicodearagon.com', 'elperiodicoextremadura.com', 'elperiodicomediterraneo.com', 'emporda.info', 'laopinioncoruna.es', 'laopiniondemalaga.es', 'laopiniondemurcia.es', 'laopiniondezamora.es', 'regio7.cat'];
+
+function init_custom_domains() {
+  au_comm_media_domains = grouped_sites['###_au_comm_media'];
+  au_thewest_domains = ['thewest.com.au'];
+  ca_gcm_domains = grouped_sites['###_ca_gcm'];
+  ca_postmedia_domains = grouped_sites['###_ca_postmedia'];
+  ch_media_domains = [];
+  cl_emol_region_domains = [];
+  de_funke_medien_domains = grouped_sites['###_de_funke_medien'];
+  de_madsack_domains = grouped_sites['###_de_madsack'];
+  es_epiberica_domains = grouped_sites['###_es_epiberica'];
+  fi_sanoma_domains = grouped_sites['###_fi_sanoma'];
+  medium_custom_domains = grouped_sites['###_medium_custom'];
+  nl_mediahuis_noord_domains = [];
+  substack_custom_domains = [];
+  usa_gannett_domains = grouped_sites['###_usa_gannett'];
+  usa_hearst_comm_domains = grouped_sites['###_usa_hearst_comm'];
+  usa_lee_ent_domains = grouped_sites['###_usa_lee_ent'];
+  usa_mcc_domains = grouped_sites['###_usa_mcc'];
+  usa_mng_domains = grouped_sites['###_usa_mng'];
+  usa_townnews_domains = [];
+}
+init_custom_domains();
+
 // sites with no fix (background)
 var nl_dpg_media_nofix_domains = ['ad.nl', 'bd.nl', 'bndestem.nl', 'destentor.nl', 'ed.nl', 'gelderlander.nl', 'hln.be', 'pzc.nl', 'tubantia.nl'];
 var nofix_sites = ['africaintelligence.com', 'africaintelligence.fr', 'aftenposten.no', 'bild.de', 'borsen.dk', 'businesstimes.com.sg', 'caixin.com', 'caixinglobal.com', 'caravanmagazine.in', 'cnbc.com', 'courrierinternational.com', 'dn.se', 'elordenmundial.com', 'epw.in', 'expresso.pt', 'ftchinese.com', 'handelsblatt.com', 'hln.be', 'ilmanifesto.it', 'ilsole24ore.com', 'jacobinmag.com', 'jeunesafrique.com', 'kurier.at', 'lavozdegalicia.es', 'leconomiste.com', 'lefigaro.fr', 'lemonde.fr', 'lepoint.fr', 'liberation.fr', 'mediapart.fr', 'milanofinanza.it', 'mondediplo.com', 'moneycontrol.com', 'nieuwsblad.be', 'nknews.org', 'ouest-france.fr', 'politiken.dk', 'publico.pt', 'republic.ru', 'rp-online.de', 'standaard.be', 'statnews.com', 'stern.de', 'straitstimes.com', 'sueddeutsche.de', 'tagesanzeiger.ch', 'tagesspiegel.de', 'the-ken.com', 'theinformation.com', ' themorningcontext.com', 'theparisreview.org', 'thewirechina.com', 'welt.de', 'worldpoliticsreview.com', 'ynet.co.il'].concat(nl_dpg_media_nofix_domains);
--- a/sites_updated.json
+++ b/sites_updated.json
@ -8,39 +8,5 @@
        "rm_elem": 1
      }
    ]
-  },
-  "Groene Amsterdammer": {
-    "domain": "groene.nl",
-    "remove_cookies_select_hold": ["accept-cookies", "popunder-hidden"],
-    "useragent": "googlebot"
-  },
-  "Substack": {
-    "domain": "substack.com",
-    "useragent": "googlebot",
-    "new_site": 1
-  },
-  "Suomen Sotilas": {
-    "domain": "suomensotilas.fi",
-    "allow_cookies": 1,
-    "block_regex": "\\/suomensotilas\\.fi\\/wp-content\\/plugins\\/epflpw\\/js\\/pw\\.js",
-    "cs_code": [{
-        "cond": "div.epfl-pw-obscured",
-        "rm_class": "epfl-pw-obscured"
-      }
-    ],
-    "new_site": 1
-  },
-  "The Athletic": {
-    "domain": "###_usa_theathletic",
-    "group": [
-      "theathletic.com"
-    ],
-    "allow_cookies": 1,
-    "block_regex": "cdn\\.ampproject\\.org\\/v\\d\\/amp-(access|subscriptions)-.+\\.js",
-    "cs_code": [{
-        "cond": "div[amp-access*=\"NOT granted\"]",
-        "rm_elem": 1
-      }
-    ]
  }
 }