From 1a4b61f7ebe0df17baeb6dfac42dcbc251fb2954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 30 Nov 2020 01:10:45 +0100 Subject: [PATCH] [downloader:http] fix issues with chunked transfer encoding (fixes #1144) --- gallery_dl/downloader/http.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index bda475ee..6580b23d 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -155,11 +155,15 @@ class HttpDownloader(DownloaderBase): size, self.maxsize) return False + chunked = response.raw.chunked + content = response.iter_content(self.chunk_size) + # check filename extension against file header if self.adjust_extension and not offset and \ pathfmt.extension in FILE_SIGNATURES: try: - file_header = next(response.iter_content(16), b"") + file_header = next( + content if chunked else response.iter_content(16), b"") except (RequestException, SSLError, OpenSSLError) as exc: msg = str(exc) print() @@ -191,7 +195,7 @@ class HttpDownloader(DownloaderBase): self.out.start(pathfmt.path) try: - self.receive(fp, response.iter_content(self.chunk_size)) + self.receive(fp, content) except (RequestException, SSLError, OpenSSLError) as exc: msg = str(exc) print()