mikf/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[deviantart] add option to minimize refresh-token usage

Browse Source 
Always trying with a public token first and repeating the API request
with a private token if deviations are missing doesn't quite work for
galleries and folders with less than 25 items, so its an option and
not the default.
This commit is contained in:
Mike Fährmann 2018-07-24 20:10:33 +02:00
parent d98e47817d
commit 886d662582
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 31 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/configuration.rst
View File

@ -346,6 +346,23 @@ Description The ``refresh_token`` value you get from linking your
=========== =====
extractor.deviantart.try-public
-------------------------------
=========== =====
Type ``bool``
Default ``false``
Description Try accessing a user's deviations with a public access token first
and only switch to a private access token if deviations are
detected as missing (i.e. they are only visible to logged in users).
This option only has an effect when using a `refresh token`__
and tries to minimize the amount of API calls with private access
tokens, as they have a much lower rate limit than public ones.
__ extractor.deviantart.refresh-token_
=========== =====
extractor.deviantart.wait-min
-----------------------------
=========== =====

22
gallery_dl/extractor/deviantart.py
View File

@ -461,6 +461,7 @@ class DeviantartAPI():
if not isinstance(self.mature, str):
self.mature = "true" if self.mature else "false"
self.try_public = extractor.config("try-public", False)
self.refresh_token = extractor.config("refresh-token")
self.client_id = extractor.config("client-id", self.CLIENT_ID)
self.client_secret = extractor.config(
@ -552,7 +553,7 @@ class DeviantartAPI():
"""Actual authenticate implementation"""
url = "https://www.deviantart.com/oauth2/token"
if refresh_token:
self.log.info("Refreshing access token")
self.log.info("Refreshing private access token")
data = {"grant_type": "refresh_token",
"refresh_token": _refresh_token_cache(refresh_token)}
else:
@ -603,16 +604,21 @@ class DeviantartAPI():
self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
def _pagination(self, endpoint, params=None):
public = self.try_public
while True:
data = self._call(endpoint, params, public=False)
if "results" in data:
yield from data["results"]
if not data["has_more"]:
return
params["offset"] = data["next_offset"]
else:
data = self._call(endpoint, params, public=public)
if "results" not in data:
self.log.error("Unexpected API response: %s", data)
return
if (public and len(data["results"]) < params["limit"] and
self.refresh_token and data["has_more"]):
self.log.info("Switching to private access token")
public = False
continue
yield from data["results"]
if not data["has_more"]:
return
params["offset"] = data["next_offset"]
def _pagination_list(self, endpoint, params=None):
result = []