mikf/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

write OAuth token to cache by default (#616)

Browse Source
This commit is contained in:
Mike Fährmann 2020-05-25 22:19:58 +02:00
parent ddc253cf9a
commit dfcf2a2c91
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
7 changed files with 70 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/configuration.rst
View File

@ -910,6 +910,16 @@ Description Controls how a user is directed to an OAuth authorization site.
=========== =====
extractor.oauth.cache
---------------------
=========== =====
Type ``bool``
Default ``true``
Description Store tokens received during OAuth authorizations
in `cache <cache.file_>`__.
=========== =====
extractor.oauth.port
--------------------
=========== =====

5
docs/gallery-dl.conf
View File

@ -30,7 +30,6 @@
},
"deviantart":
{
"refresh-token": null,
"extra": false,
"flat": true,
"folders": false,
@ -51,8 +50,6 @@
},
"flickr":
{
"access-token": null,
"access-token-secret": null,
"videos": true,
"size-max": null
},
@ -96,6 +93,7 @@
"oauth":
{
"browser": true,
"cache": true,
"port": 6414
},
"pixiv":
@ -120,7 +118,6 @@
},
"reddit":
{
"refresh-token": null,
"comments": 0,
"morecomments": false,
"date-min": 0,

37
gallery_dl/extractor/deviantart.py
View File

@ -850,9 +850,12 @@ class DeviantartOAuthAPI():
self.client_secret = extractor.config(
"client-secret", self.CLIENT_SECRET)
self.refresh_token = extractor.config("refresh-token")
if self.refresh_token == "cache":
self.refresh_token = "#" + str(self.client_id)
token = extractor.config("refresh-token")
if token is None or token == "cache":
token = "#" + str(self.client_id)
if not _refresh_token_cache(token):
token = None
self.refresh_token_key = token
self.log.debug(
"Using %s API credentials (client-id %s)",
@ -952,18 +955,19 @@ class DeviantartOAuthAPI():
endpoint = "user/profile/" + username
return self._call(endpoint, fatal=False)
def authenticate(self, refresh_token):
def authenticate(self, refresh_token_key):
"""Authenticate the application by requesting an access token"""
self.headers["Authorization"] = self._authenticate_impl(refresh_token)
self.headers["Authorization"] = \
self._authenticate_impl(refresh_token_key)
@cache(maxage=3600, keyarg=1)
def _authenticate_impl(self, refresh_token):
def _authenticate_impl(self, refresh_token_key):
"""Actual authenticate implementation"""
url = "https://www.deviantart.com/oauth2/token"
if refresh_token:
if refresh_token_key:
self.log.info("Refreshing private access token")
data = {"grant_type": "refresh_token",
"refresh_token": _refresh_token_cache(refresh_token)}
"refresh_token": _refresh_token_cache(refresh_token_key)}
else:
self.log.info("Requesting public access token")
data = {"grant_type": "client_credentials"}
@ -977,8 +981,9 @@ class DeviantartOAuthAPI():
self.log.debug("Server response: %s", data)
raise exception.AuthenticationError('"{}" ({})'.format(
data.get("error_description"), data.get("error")))
if refresh_token:
_refresh_token_cache.update(refresh_token, data["refresh_token"])
if refresh_token_key:
_refresh_token_cache.update(
refresh_token_key, data["refresh_token"])
return "Bearer " + data["access_token"]
def _call(self, endpoint, params=None, fatal=True, public=True):
@ -988,7 +993,7 @@ class DeviantartOAuthAPI():
if self.delay >= 0:
time.sleep(2 ** self.delay)
self.authenticate(None if public else self.refresh_token)
self.authenticate(None if public else self.refresh_token_key)
response = self.extractor.request(
url, headers=self.headers, params=params, fatal=None)
data = response.json()
@ -1024,7 +1029,7 @@ class DeviantartOAuthAPI():
if extend:
if public and len(data["results"]) < params["limit"]:
if self.refresh_token:
if self.refresh_token_key:
self.log.debug("Switching to private access token")
public = False
continue
@ -1155,9 +1160,11 @@ class DeviantartEclipseAPI():
return text.rextract(page, '\\"id\\":', ',', pos)[0].strip('" ')
@cache(maxage=10*365*24*3600, keyarg=0)
def _refresh_token_cache(original_token, new_token=None):
return new_token or original_token
@cache(maxage=100*365*24*3600, keyarg=0)
def _refresh_token_cache(token):
if token and token[0] == "#":
return None
return token
###############################################################################

11
gallery_dl/extractor/oauth.py
View File

@ -26,6 +26,7 @@ class OAuthBase(Extractor):
def __init__(self, match):
Extractor.__init__(self, match)
self.client = None
self.cache = config.get(("extractor", self.category), "cache", True)
def oauth_config(self, key, default=None):
return config.interpolate(
@ -94,6 +95,13 @@ class OAuthBase(Extractor):
token_secret=data["oauth_token_secret"],
))
# write to cache
if self.cache:
key = (self.subcategory, self.session.auth.consumer_key)
tokens = (data["oauth_token"], data["oauth_token_secret"])
oauth._token_cache.update(key, tokens)
self.log.info("Writing tokens to cache")
def _oauth2_authorization_code_grant(
self, client_id, client_secret, auth_url, token_url,
scope="read", key="refresh_token", auth=True,
@ -162,7 +170,7 @@ class OAuthBase(Extractor):
))
# write to cache
if cache and config.get(("extractor", self.category), "cache"):
if self.cache and cache:
cache.update("#" + str(client_id), data[key])
self.log.info("Writing 'refresh-token' to cache")
@ -223,6 +231,7 @@ class OAuthReddit(OAuthBase):
"https://www.reddit.com/api/v1/authorize",
"https://www.reddit.com/api/v1/access_token",
scope="read history",
cache=reddit._refresh_token_cache,
)

15
gallery_dl/extractor/reddit.py
View File

@ -222,7 +222,6 @@ class RedditAPI():
self.extractor = extractor
self.comments = text.parse_int(extractor.config("comments", 0))
self.morecomments = extractor.config("morecomments", False)
self.refresh_token = extractor.config("refresh-token")
self.log = extractor.log
client_id = extractor.config("client-id", self.CLIENT_ID)
@ -236,6 +235,13 @@ class RedditAPI():
self.client_id = client_id
self.headers = {"User-Agent": user_agent}
token = extractor.config("refresh-token")
if token is None or token == "cache":
key = "#" + self.client_id
self.refresh_token = _refresh_token_cache(key)
else:
self.refresh_token = token
def submission(self, submission_id):
"""Fetch the (submission, comments)=-tuple for a submission id"""
endpoint = "/comments/" + submission_id + "/.json"
@ -382,3 +388,10 @@ class RedditAPI():
@staticmethod
def _decode(sid):
return util.bdecode(sid, "0123456789abcdefghijklmnopqrstuvwxyz")
@cache(maxage=100*365*24*3600, keyarg=0)
def _refresh_token_cache(token):
if token and token[0] == "#":
return None
return token

12
gallery_dl/oauth.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2018-2019 Mike Fährmann
# Copyright 2018-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@ -20,6 +20,7 @@ import requests
import requests.auth
from . import text
from .cache import cache
def nonce(size, alphabet=string.ascii_letters):
@ -117,6 +118,10 @@ class OAuth1API():
token_secret = extractor.config("access-token-secret")
key_type = "default" if api_key == self.API_KEY else "custom"
if token is None or token == "cache":
key = (extractor.category, api_key)
token, token_secret = _token_cache(key)
if api_key and api_secret and token and token_secret:
self.log.debug("Using %s OAuth1.0 authentication", key_type)
self.session = OAuth1Session(
@ -131,3 +136,8 @@ class OAuth1API():
kwargs["fatal"] = None
kwargs["session"] = self.session
return self.extractor.request(url, **kwargs)
@cache(maxage=100*365*24*3600, keyarg=0)
def _token_cache(key):
return None, None

4
test/test_cache.py
View File

@ -22,8 +22,8 @@ config.set(("cache",), "file", dbpath)
from gallery_dl import cache # noqa E402
def tearDownModule():
util.remove_file(dbpath)
# def tearDownModule():
# util.remove_file(dbpath)
class TestCache(unittest.TestCase):