Only use Googlebot for a certain set of sites instead of being on by default

background.js

@@ -143,6 +143,13 @@ const remove_cookies = [
 'handelsblatt.com',
 ]
 
+// Override User-Agent with Googlebot
+const use_google_bot = [
+'theaustralian.com.au',
+'telegraph.co.uk',
+'thetimes.co.uk',
+]
+
 function setDefaultOptions() {
   browser.storage.sync.set({
     sites: defaultSites
@@ -151,7 +158,6 @@ function setDefaultOptions() {
   });
 }
 
-
 const blockedRegexes = [
 /.+:\/\/.+\.tribdss\.com\//,
 /thenation\.com\/.+\/paywall-script\.php/,
@@ -270,18 +276,21 @@ browser.webRequest.onBeforeSendHeaders.addListener(function(details) {
     }
   }
 
-  // override User-Agent except on medium.com and thesaturdaypaper.com.au
-  if (details.url.indexOf("medium.com") === -1 && details.url.indexOf("thesaturdaypaper.com.au") === -1) {
+  // override User-Agent to use Googlebot
+  var useGoogleBot = use_google_bot.filter(function(item) {
+    return typeof item == 'string' && details.url.indexOf(item) > -1;            
+  }).length > 0;
+
+  if (useGoogleBot) {
     requestHeaders.push({
       "name": "User-Agent",
       "value": useUserAgentMobile ? userAgentMobile : userAgentDesktop
     })
-  }
-
     requestHeaders.push({
       "name": "X-Forwarded-For",
       "value": "66.249.66.1"
     })
+  }
 
   // remove cookies before page load
   requestHeaders = requestHeaders.map(function(requestHeader) {