From 886d6625826a9da7e8a2607e3cec9b40ed771e41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Tue, 24 Jul 2018 20:10:33 +0200
Subject: [PATCH] [deviantart] add option to minimize refresh-token usage

Always trying with a public token first and repeating the API request
with a private token if deviations are missing doesn't quite work for
galleries and folders with less than 25 items, so its an option and
not the default.
---
 docs/configuration.rst             | 17 +++++++++++++++++
 gallery_dl/extractor/deviantart.py | 22 ++++++++++++++--------
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index fd65321a..90866f09 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -346,6 +346,23 @@ Description The ``refresh_token`` value you get from linking your
 =========== =====
 
 
+extractor.deviantart.try-public
+-------------------------------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Try accessing a user's deviations with a public access token first
+            and only switch to a private access token if deviations are
+            detected as missing (i.e. they are only visible to logged in users).
+
+            This option only has an effect when using a `refresh token`__
+            and tries to minimize the amount of API calls with private access
+            tokens, as they have a much lower rate limit than public ones.
+
+            __ extractor.deviantart.refresh-token_
+=========== =====
+
+
 extractor.deviantart.wait-min
 -----------------------------
 =========== =====
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index acb3e797..65c6d55c 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -461,6 +461,7 @@ class DeviantartAPI():
         if not isinstance(self.mature, str):
             self.mature = "true" if self.mature else "false"
 
+        self.try_public = extractor.config("try-public", False)
         self.refresh_token = extractor.config("refresh-token")
         self.client_id = extractor.config("client-id", self.CLIENT_ID)
         self.client_secret = extractor.config(
@@ -552,7 +553,7 @@ class DeviantartAPI():
         """Actual authenticate implementation"""
         url = "https://www.deviantart.com/oauth2/token"
         if refresh_token:
-            self.log.info("Refreshing access token")
+            self.log.info("Refreshing private access token")
             data = {"grant_type": "refresh_token",
                     "refresh_token": _refresh_token_cache(refresh_token)}
         else:
@@ -603,16 +604,21 @@ class DeviantartAPI():
                 self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
 
     def _pagination(self, endpoint, params=None):
+        public = self.try_public
         while True:
-            data = self._call(endpoint, params, public=False)
-            if "results" in data:
-                yield from data["results"]
-                if not data["has_more"]:
-                    return
-                params["offset"] = data["next_offset"]
-            else:
+            data = self._call(endpoint, params, public=public)
+            if "results" not in data:
                 self.log.error("Unexpected API response: %s", data)
                 return
+            if (public and len(data["results"]) < params["limit"] and
+                    self.refresh_token and data["has_more"]):
+                self.log.info("Switching to private access token")
+                public = False
+                continue
+            yield from data["results"]
+            if not data["has_more"]:
+                return
+            params["offset"] = data["next_offset"]
 
     def _pagination_list(self, endpoint, params=None):
         result = []