mikf/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use 'retries' and 'timeout' options for regular HTTP requests

Browse Source
This commit is contained in:
Mike Fährmann 2018-08-02 14:59:37 +02:00
parent f3793660ef
commit 68d6033a5d
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
1 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
gallery_dl/extractor/common.py
View File

@ -37,6 +37,10 @@ class Extractor():
self._set_headers() self._set_headers()
self._set_cookies() self._set_cookies()
self._set_proxies() self._set_proxies()
self._retries = config.interpolate(
("downloader", "http", "retries"), 5)
self._timeout = config.interpolate(
("downloader", "http", "timeout"), 30)
def __iter__(self): def __iter__(self):
return self.items() return self.items()
@ -51,12 +55,14 @@ class Extractor():
return config.interpolate( return config.interpolate(
("extractor", self.category, self.subcategory, key), default) ("extractor", self.category, self.subcategory, key), default)
def request(self, url, method="GET", encoding=None, expect=(), retries=3, def request(self, url, method="GET", *,
*args, **kwargs): encoding=None, expect=(), retries=None, **kwargs):
max_tries = retries tries = 0
retries = retries or self._retries
kwargs.setdefault("timeout", self._timeout)
while True: while True:
try: try:
response = self.session.request(method, url, *args, **kwargs) response = self.session.request(method, url, **kwargs)
except (requests.ConnectionError, requests.Timeout) as exc: except (requests.ConnectionError, requests.Timeout) as exc:
msg = exc msg = exc
except requests.exceptions.RequestException as exc: except requests.exceptions.RequestException as exc:
@ -73,12 +79,11 @@ class Extractor():
if code < 500 and code != 429: if code < 500 and code != 429:
break break
if not retries: if tries >= retries:
break break
tries = max_tries - retries self.log.debug("%s (%d/%d)", msg, tries + 1, retries)
retries -= 1
self.log.debug("%s (%d/%d)", msg, tries, max_tries)
time.sleep(2 ** tries) time.sleep(2 ** tries)
tries += 1
raise exception.HttpError(msg) raise exception.HttpError(msg)