use 'Content-Length' to determine incomplete downloads (#29)

This commit is contained in:
Mike Fährmann 2017-10-20 18:56:18 +02:00
parent 16783e327f
commit 2e982f56af
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 21 additions and 2 deletions

View File

@ -13,7 +13,7 @@ import requests.exceptions as rexcepts
import mimetypes
import logging
from .common import BasicDownloader
from .. import config
from .. import config, util
log = logging.getLogger("http")
@ -30,8 +30,10 @@ class Downloader(BasicDownloader):
self.out = output
def download_impl(self, url, pathfmt):
partial = False
tries = 0
msg = ""
while True:
tries += 1
if tries > 1:
@ -53,7 +55,7 @@ class Downloader(BasicDownloader):
break
# reject error-status-codes
if response.status_code != 200:
if response.status_code not in (200, 206):
msg = 'HTTP status "{} {}"'.format(
response.status_code, response.reason
)
@ -79,6 +81,13 @@ class Downloader(BasicDownloader):
response.close()
return
#
if partial and "Content-Range" in response.headers:
size = response.headers["Content-Range"].rpartition("/")[2]
else:
size = response.headers.get("Content-Length")
size = util.safe_int(size)
# everything ok -- proceed to download
self.out.start(pathfmt.path)
self.downloading = True
@ -86,6 +95,10 @@ class Downloader(BasicDownloader):
with pathfmt.open() as file:
for data in response.iter_content(16384):
file.write(data)
if size and file.tell() != size:
msg = "filesize mismatch ({} != {})".format(
file.tell(), size)
continue
except rexcepts.RequestException as exception:
msg = exception
response.close()

View File

@ -255,6 +255,7 @@ class TestJob(DownloadJob):
def __init__(self, hashobj):
self.hashobj = hashobj
self.path = ""
self.size = 0
self.has_extension = True
def __enter__(self):
@ -264,12 +265,17 @@ class TestJob(DownloadJob):
pass
def open(self):
self.size = 0
return self
def write(self, content):
"""Update SHA1 hash"""
self.size += len(content)
self.hashobj.update(content)
def tell(self):
return self.size
def __init__(self, url, parent=None, content=False):
DownloadJob.__init__(self, url, parent)
self.content = content