anime-downloader/anime_downloader/downloader/http_downloader.py

import os
import copy
import logging

from anime_downloader.downloader.base_downloader import BaseDownloader
from anime_downloader import session
import requests
import requests_cache

session = session.get_session()
session = requests
logger = logging.getLogger(__name__)


class HTTPDownloader(BaseDownloader):
    def _download(self):
        logger.warning('Using internal downloader which might be slow. Use aria2 for full bandwidth.')
        if self.range_size is None:
            self._non_range_download()
        else:
            self._ranged_download()

    def _ranged_download(self):
        http_chunksize = self.range_size

        range_start = 0
        range_end = http_chunksize

        url = self.source.stream_url
        headers = self.source.headers
        if 'user-agent' not in headers:
            headers['user-agent'] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0.1) Gecko/20100101 Firefox/88.0.1"

        if self.source.referer:
            headers['Referer'] = self.source.referer

        # Make a new file, maybe not the best way
        with open(self.path, 'w'):
            pass

        r = session.get(url, headers=headers, stream=True)
        while self.downloaded < self._total_size:
            r = session.get(url,
                            headers=set_range(range_start, range_end, headers),
                            stream=True)
            if r.status_code == 206:
                with open(self.path, 'ab') as f:
                    for chunk in r.iter_content(chunk_size=self.chunksize):
                        if chunk:
                            f.write(chunk)
                            self.report_chunk_downloaded()

            if range_end == '':
                break
            range_start = os.stat(self.path).st_size
            range_end += http_chunksize
            if range_end > self._total_size:
                range_end = ''

    def _non_range_download(self):
        url = self.source.stream_url
        headers = {
            'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0.1) Gecko/20100101 Firefox/88.0.1"
        }
        if self.source.referer:
            headers['Referer'] = self.source.referer
        r = session.get(url, headers=headers, stream=True)

        if r.status_code == 200:
            with open(self.path, 'wb') as f:
                for chunk in r.iter_content(chunk_size=self.chunksize):
                    if chunk:
                        f.write(chunk)
                        self.report_chunk_downloaded()


def set_range(start=0, end='', headers=None):
    if headers is None:
        headers = {}
    headers = copy.copy(headers)

    headers['Range'] = 'bytes={}-{}'.format(start, end)
    return headers