use 'Content-Length' to determine incomplete downloads (#29)

gallery_dl/downloader/http.py

@@ -13,7 +13,7 @@ import requests.exceptions as rexcepts
 import mimetypes
 import logging
 from .common import BasicDownloader
-from .. import config
+from .. import config, util
 
 log = logging.getLogger("http")
 
@@ -30,8 +30,10 @@ class Downloader(BasicDownloader):
         self.out = output
 
     def download_impl(self, url, pathfmt):
+        partial = False
         tries = 0
         msg = ""
+
         while True:
             tries += 1
             if tries > 1:
@@ -53,7 +55,7 @@ class Downloader(BasicDownloader):
                 break
 
             # reject error-status-codes
-            if response.status_code != 200:
+            if response.status_code not in (200, 206):
                 msg = 'HTTP status "{} {}"'.format(
                     response.status_code, response.reason
                 )
@@ -79,6 +81,13 @@ class Downloader(BasicDownloader):
                     response.close()
                     return
 
+            #
+            if partial and "Content-Range" in response.headers:
+                size = response.headers["Content-Range"].rpartition("/")[2]
+            else:
+                size = response.headers.get("Content-Length")
+            size = util.safe_int(size)
+
             # everything ok -- proceed to download
             self.out.start(pathfmt.path)
             self.downloading = True
@@ -86,6 +95,10 @@ class Downloader(BasicDownloader):
                 with pathfmt.open() as file:
                     for data in response.iter_content(16384):
                         file.write(data)
+                    if size and file.tell() != size:
+                        msg = "filesize mismatch ({} != {})".format(
+                            file.tell(), size)
+                        continue
             except rexcepts.RequestException as exception:
                 msg = exception
                 response.close()

gallery_dl/job.py

@@ -255,6 +255,7 @@ class TestJob(DownloadJob):
         def __init__(self, hashobj):
             self.hashobj = hashobj
             self.path = ""
+            self.size = 0
             self.has_extension = True
 
         def __enter__(self):
@@ -264,12 +265,17 @@ class TestJob(DownloadJob):
             pass
 
         def open(self):
+            self.size = 0
             return self
 
         def write(self, content):
             """Update SHA1 hash"""
+            self.size += len(content)
             self.hashobj.update(content)
 
+        def tell(self):
+            return self.size
+
     def __init__(self, url, parent=None, content=False):
         DownloadJob.__init__(self, url, parent)
         self.content = content