mikf/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add 'ciphers' option; update default User-Agent

Browse Source
This commit is contained in:
Mike Fährmann 2019-07-18 20:06:21 +02:00
parent 84f4d3bc0b
commit 21991acc49
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
3 changed files with 50 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/configuration.rst
View File

@ -217,7 +217,7 @@ extractor.*.user-agent
----------------------
=========== =====
Type ``string``
Default ``"Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0"``
Default ``"Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0"``
Description User-Agent header value to be used for HTTP requests.
Note: This option has no effect on `pixiv` and
@ -1457,6 +1457,19 @@ Description Path of the SQLite3 database used to cache login sessions,
=========== =====
ciphers
-------
=========== =====
Type ``bool`` or ``string``
Default ``true``
Description * ``true``: Update urllib3's default cipher list
* ``false``: Leave the default cipher list as is
* Any ``string``: Replace urllib3's default ciphers with these
(See `SSLContext.set_ciphers() <https://docs.python.org/3/library/ssl.html#ssl.SSLContext.set_ciphers>`__
for details)
=========== =====
API Tokens & IDs
================

2
docs/gallery-dl.conf
View File

@ -8,7 +8,7 @@
"proxy": null,
"skip": true,
"sleep": 0,
"user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0",
"user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0",
"artstation":
{

57
gallery_dl/extractor/common.py
View File

@ -145,8 +145,8 @@ class Extractor():
headers.clear()
headers["User-Agent"] = self.config(
"user-agent", ("Mozilla/5.0 (X11; Linux x86_64; rv:62.0) "
"Gecko/20100101 Firefox/62.0"))
"user-agent", ("Mozilla/5.0 (X11; Linux x86_64; rv:68.0) "
"Gecko/20100101 Firefox/68.0"))
headers["Accept"] = "*/*"
headers["Accept-Language"] = "en-US,en;q=0.5"
headers["Accept-Encoding"] = "gzip, deflate"
@ -432,23 +432,36 @@ def generate_extractors(extractor_data, symtable, classes):
http.cookiejar.MozillaCookieJar.magic_re = re.compile(
"#( Netscape)? HTTP Cookie File", re.IGNORECASE)
# Replace default cipher list of urllib3
# with the one used by Firefox to avoid Cloudflare CAPTCHAs
from requests.packages.urllib3.util import ssl_ # noqa
ssl_.DEFAULT_CIPHERS = (
"ECDHE-ECDSA-AES128-GCM-SHA256:"
"ECDHE-RSA-AES128-GCM-SHA256:"
"ECDHE-ECDSA-CHACHA20-POLY1305:"
"ECDHE-RSA-CHACHA20-POLY1305:"
"ECDHE-ECDSA-AES256-GCM-SHA384:"
"ECDHE-RSA-AES256-GCM-SHA384:"
"ECDHE-ECDSA-AES256-SHA:"
"ECDHE-ECDSA-AES128-SHA:"
"ECDHE-RSA-AES128-SHA:"
"ECDHE-RSA-AES256-SHA:"
"DHE-RSA-AES128-SHA:"
"DHE-RSA-AES256-SHA:"
"AES128-SHA:"
"AES256-SHA:"
"DES-CBC3-SHA"
)
# Replace default cipher list of urllib3 to avoid Cloudflare CAPTCHAs
ciphers = config.get(("ciphers",), True)
if ciphers:
logging.getLogger("gallery-dl").debug("Updating urllib3 ciphers")
if ciphers is True:
ciphers = (
# Firefox's list
"TLS_AES_128_GCM_SHA256:"
"TLS_CHACHA20_POLY1305_SHA256:"
"TLS_AES_256_GCM_SHA384:"
"ECDHE-ECDSA-AES128-GCM-SHA256:"
"ECDHE-RSA-AES128-GCM-SHA256:"
"ECDHE-ECDSA-CHACHA20-POLY1305:"
"ECDHE-RSA-CHACHA20-POLY1305:"
"ECDHE-ECDSA-AES256-GCM-SHA384:"
"ECDHE-RSA-AES256-GCM-SHA384:"
"ECDHE-ECDSA-AES256-SHA:"
"ECDHE-ECDSA-AES128-SHA:"
"ECDHE-RSA-AES128-SHA:"
"ECDHE-RSA-AES256-SHA:"
"DHE-RSA-AES128-SHA:"
"DHE-RSA-AES256-SHA:"
"AES128-SHA:"
"AES256-SHA:"
"DES-CBC3-SHA"
)
elif isinstance(ciphers, list):
ciphers = ":".join(ciphers)
from requests.packages.urllib3.util import ssl_ # noqa
ssl_.DEFAULT_CIPHERS = ciphers
del ssl_