use 'Content-Length' to determine incomplete downloads (#29)
This commit is contained in:
parent
16783e327f
commit
2e982f56af
@ -13,7 +13,7 @@ import requests.exceptions as rexcepts
|
||||
import mimetypes
|
||||
import logging
|
||||
from .common import BasicDownloader
|
||||
from .. import config
|
||||
from .. import config, util
|
||||
|
||||
log = logging.getLogger("http")
|
||||
|
||||
@ -30,8 +30,10 @@ class Downloader(BasicDownloader):
|
||||
self.out = output
|
||||
|
||||
def download_impl(self, url, pathfmt):
|
||||
partial = False
|
||||
tries = 0
|
||||
msg = ""
|
||||
|
||||
while True:
|
||||
tries += 1
|
||||
if tries > 1:
|
||||
@ -53,7 +55,7 @@ class Downloader(BasicDownloader):
|
||||
break
|
||||
|
||||
# reject error-status-codes
|
||||
if response.status_code != 200:
|
||||
if response.status_code not in (200, 206):
|
||||
msg = 'HTTP status "{} {}"'.format(
|
||||
response.status_code, response.reason
|
||||
)
|
||||
@ -79,6 +81,13 @@ class Downloader(BasicDownloader):
|
||||
response.close()
|
||||
return
|
||||
|
||||
#
|
||||
if partial and "Content-Range" in response.headers:
|
||||
size = response.headers["Content-Range"].rpartition("/")[2]
|
||||
else:
|
||||
size = response.headers.get("Content-Length")
|
||||
size = util.safe_int(size)
|
||||
|
||||
# everything ok -- proceed to download
|
||||
self.out.start(pathfmt.path)
|
||||
self.downloading = True
|
||||
@ -86,6 +95,10 @@ class Downloader(BasicDownloader):
|
||||
with pathfmt.open() as file:
|
||||
for data in response.iter_content(16384):
|
||||
file.write(data)
|
||||
if size and file.tell() != size:
|
||||
msg = "filesize mismatch ({} != {})".format(
|
||||
file.tell(), size)
|
||||
continue
|
||||
except rexcepts.RequestException as exception:
|
||||
msg = exception
|
||||
response.close()
|
||||
|
@ -255,6 +255,7 @@ class TestJob(DownloadJob):
|
||||
def __init__(self, hashobj):
|
||||
self.hashobj = hashobj
|
||||
self.path = ""
|
||||
self.size = 0
|
||||
self.has_extension = True
|
||||
|
||||
def __enter__(self):
|
||||
@ -264,12 +265,17 @@ class TestJob(DownloadJob):
|
||||
pass
|
||||
|
||||
def open(self):
|
||||
self.size = 0
|
||||
return self
|
||||
|
||||
def write(self, content):
|
||||
"""Update SHA1 hash"""
|
||||
self.size += len(content)
|
||||
self.hashobj.update(content)
|
||||
|
||||
def tell(self):
|
||||
return self.size
|
||||
|
||||
def __init__(self, url, parent=None, content=False):
|
||||
DownloadJob.__init__(self, url, parent)
|
||||
self.content = content
|
||||
|
Loading…
x
Reference in New Issue
Block a user