diff --git a/anime_downloader/cli.py b/anime_downloader/cli.py index 4a3dc05..7235185 100644 --- a/anime_downloader/cli.py +++ b/anime_downloader/cli.py @@ -4,6 +4,7 @@ import os import logging +from anime_downloader.session import session from anime_downloader.sites import get_anime_class from anime_downloader.players.mpv import mpv from anime_downloader.__version__ import __version__ @@ -95,10 +96,7 @@ def dl(ctx, anime_url, episode_range, url, player, skip_download, quality, cls = get_anime_class(anime_url) disable_ssl = cls and cls.__name__ == 'Masterani' or disable_ssl - util.set_default_requests_options( - # Set global options for requests here. For example: timeout=N, proxies={} - verify=not disable_ssl - ) + session.verify = not disable_ssl if not cls: anime_url = util.search(anime_url, provider) diff --git a/anime_downloader/downloader/base_downloader.py b/anime_downloader/downloader/base_downloader.py index 6223cd9..006b65f 100644 --- a/anime_downloader/downloader/base_downloader.py +++ b/anime_downloader/downloader/base_downloader.py @@ -5,6 +5,7 @@ import logging import sys from anime_downloader import util +from anime_downloader.session import session class BaseDownloader: @@ -20,7 +21,7 @@ class BaseDownloader: self.chunksize = 16384 - r = requests.get(self.url, stream=True, **util.get_requests_options()) + r = session.get(self.url, stream=True) self.total_size = int(r.headers['Content-length']) if os.path.exists(path): diff --git a/anime_downloader/downloader/http_downloader.py b/anime_downloader/downloader/http_downloader.py index 58019fe..346ada7 100644 --- a/anime_downloader/downloader/http_downloader.py +++ b/anime_downloader/downloader/http_downloader.py @@ -3,6 +3,7 @@ import os from anime_downloader import util from anime_downloader.downloader.base_downloader import BaseDownloader +from anime_downloader.session import session class HTTPDownloader(BaseDownloader): @@ -22,12 +23,11 @@ class HTTPDownloader(BaseDownloader): with open(self.path, 'w'): pass - r = requests.get(self.url, stream=True, **util.get_requests_options()) + r = session.get(self.url, stream=True) while self.downloaded < self.total_size: - r = requests.get(self.url, - headers=set_range(range_start, range_end), - stream=True, - **util.get_requests_options()) + r = session.get(self.url, + headers=set_range(range_start, range_end), + stream=True) if r.status_code == 206: with open(self.path, 'ab') as f: for chunk in r.iter_content(chunk_size=self.chunksize): @@ -43,7 +43,7 @@ class HTTPDownloader(BaseDownloader): range_end = '' def _non_range_download(self): - r = requests.get(self.url, stream=True, **util.get_requests_options()) + r = session.get(self.url, stream=True) if r.status_code == 200: with open(self.path, 'wb') as f: diff --git a/anime_downloader/extractors/moe.py b/anime_downloader/extractors/moe.py index 4a1dc7c..facfb61 100644 --- a/anime_downloader/extractors/moe.py +++ b/anime_downloader/extractors/moe.py @@ -4,12 +4,13 @@ import base64 from anime_downloader import util from anime_downloader.extractors.base_extractor import BaseExtractor +from anime_downloader.session import session class StreamMoe(BaseExtractor): def _get_data(self): url = self.url - res = requests.get(url, **util.get_requests_options()) + res = session.get(url) content_re = re.compile(r"= atob\('(.*?)'\)") source_re = re.compile(r'source src="(.*?)"') diff --git a/anime_downloader/extractors/mp4upload.py b/anime_downloader/extractors/mp4upload.py index 564f6d9..f46bdf6 100644 --- a/anime_downloader/extractors/mp4upload.py +++ b/anime_downloader/extractors/mp4upload.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup from anime_downloader import util from anime_downloader.extractors.base_extractor import BaseExtractor +from anime_downloader.session import session class MP4Upload(BaseExtractor): @@ -21,7 +22,7 @@ class MP4Upload(BaseExtractor): r'.*?(www\d+).*?\|video\|(.*?)\|(\d+)\|.*?', re.DOTALL) - mp4u_embed = requests.get(self.url, **util.get_requests_options()).text + mp4u_embed = session.get(self.url).text domain, video_id, protocol = source_parts_re.match(mp4u_embed).groups() logging.debug('Domain: %s, Video ID: %s, Protocol: %s' % @@ -29,7 +30,7 @@ class MP4Upload(BaseExtractor): url = self.url.replace('embed-', '') # Return to non-embed page to collect title - mp4u_page = BeautifulSoup(requests.get(url, **util.get_requests_options()).text, 'html.parser') + mp4u_page = BeautifulSoup(session.get(url).text, 'html.parser') title = mp4u_page.find('span', {'class': 'dfilename'}).text title = title[:title.rfind('_')][:title.rfind('.')].replace(' ', '_') diff --git a/anime_downloader/extractors/rapidvideo.py b/anime_downloader/extractors/rapidvideo.py index 2b8bd3f..ef3579d 100644 --- a/anime_downloader/extractors/rapidvideo.py +++ b/anime_downloader/extractors/rapidvideo.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup from anime_downloader import util from anime_downloader.extractors.base_extractor import BaseExtractor +from anime_downloader.session import session class RapidVideo(BaseExtractor): @@ -14,13 +15,13 @@ class RapidVideo(BaseExtractor): headers = self.headers headers['referer'] = url try: - r = requests.get(url, headers=headers, **util.get_requests_options()) + r = session.get(url, headers=headers) except: - r = requests.post(url, { + r = session.post(url, { 'cursor.x': 12, 'cursor.y': 12, 'block': 1, - }, headers=headers, **util.get_requests_options()) + }, headers=headers) soup = BeautifulSoup(r.text, 'html.parser') # TODO: Make these a different function. Can be reused in other classes diff --git a/anime_downloader/session.py b/anime_downloader/session.py new file mode 100644 index 0000000..d733b8a --- /dev/null +++ b/anime_downloader/session.py @@ -0,0 +1,3 @@ +import requests + +session = requests.Session() diff --git a/anime_downloader/sites/anime.py b/anime_downloader/sites/anime.py index 8e767c0..58ce775 100644 --- a/anime_downloader/sites/anime.py +++ b/anime_downloader/sites/anime.py @@ -7,6 +7,7 @@ import logging import sys import copy +from anime_downloader.session import session from anime_downloader.sites.exceptions import AnimeDLError, NotFoundError from anime_downloader import util from anime_downloader.const import desktop_headers @@ -49,7 +50,7 @@ class BaseAnime: def get_data(self): self._episode_urls = [] - r = requests.get(self.url, headers=desktop_headers, **util.get_requests_options()) + r = session.get(self.url, headers=desktop_headers) soup = BeautifulSoup(r.text, 'html.parser') try: diff --git a/anime_downloader/sites/gogoanime.py b/anime_downloader/sites/gogoanime.py index 8ee11e8..9656f6d 100644 --- a/anime_downloader/sites/gogoanime.py +++ b/anime_downloader/sites/gogoanime.py @@ -1,4 +1,5 @@ from anime_downloader import util +from anime_downloader.session import session from anime_downloader.sites.anime import BaseAnime, BaseEpisode import requests import re @@ -10,7 +11,7 @@ class GogoanimeEpisode(BaseEpisode): _base_url = 'https://www2.gogoanime.se' def _get_sources(self): - soup = BeautifulSoup(requests.get(self.url, **util.get_requests_options()).text, 'html.parser') + soup = BeautifulSoup(session.get(self.url).text, 'html.parser') url = 'https:'+soup.select_one('li.anime a').get('data-video') res = requests.get(url) @@ -35,7 +36,7 @@ class GogoAnime(BaseAnime): 'id': anime_id, } - res = requests.get(self._episode_list_url, params=params, **util.get_requests_options()) + res = session.get(self._episode_list_url, params=params) soup = BeautifulSoup(res.text, 'html.parser') epurls = list( diff --git a/anime_downloader/sites/kisscartoon.py b/anime_downloader/sites/kisscartoon.py index 601a95b..e4cf50a 100644 --- a/anime_downloader/sites/kisscartoon.py +++ b/anime_downloader/sites/kisscartoon.py @@ -1,4 +1,5 @@ from anime_downloader import util +from anime_downloader.session import session from anime_downloader.sites.kissanime import KissAnime from anime_downloader.sites.anime import BaseEpisode, SearchResult from anime_downloader.sites.exceptions import NotFoundError @@ -25,13 +26,12 @@ class KisscartoonEpisode(BaseEpisode): } headers = desktop_headers headers['referer'] = self.url - res = requests.get(self._episode_list_url, - params=params, headers=headers, **util.get_requests_options()) + res = session.get(self._episode_list_url, params=params, headers=headers) url = res.json()['value'] headers = desktop_headers headers['referer'] = self.url - res = requests.get('https:' + url, headers=headers, **util.get_requests_options()) + res = session.get('https:' + url, headers=headers) return [( 'no_extractor', diff --git a/anime_downloader/sites/nineanime.py b/anime_downloader/sites/nineanime.py index fd254d2..fee6240 100644 --- a/anime_downloader/sites/nineanime.py +++ b/anime_downloader/sites/nineanime.py @@ -1,3 +1,4 @@ +from anime_downloader.session import session from anime_downloader.sites.anime import BaseAnime, BaseEpisode, SearchResult from anime_downloader.sites.exceptions import NotFoundError, AnimeDLError from anime_downloader import util @@ -59,8 +60,7 @@ class NineAnime(BaseAnime): @classmethod def search(cls, query): - r = requests.get('https://www4.9anime.is/search?', - params={'keyword': query}, headers=desktop_headers, **util.get_requests_options()) + r = session.get('https://www4.9anime.is/search?', params={'keyword': query}, headers=desktop_headers) logging.debug(r.url) @@ -104,7 +104,7 @@ class NineAnime(BaseAnime): params = {} params['_'] = int(generate_(params)) params['_'] = 648 - soup = BeautifulSoup(requests.get(api_url, params=params, **util.get_requests_options()).json()['html'], 'html.parser') + soup = BeautifulSoup(session.get(api_url, params=params).json()['html'], 'html.parser') episodes = soup.find('div', {'class': 'server', 'data-name': 33}) episodes = episodes.find_all('li') diff --git a/anime_downloader/sites/twistmoe.py b/anime_downloader/sites/twistmoe.py index 7224268..ec645c8 100644 --- a/anime_downloader/sites/twistmoe.py +++ b/anime_downloader/sites/twistmoe.py @@ -7,6 +7,7 @@ from bs4 import BeautifulSoup import warnings from anime_downloader import util +from anime_downloader.session import session from anime_downloader.sites.anime import BaseAnime, BaseEpisode, SearchResult @@ -34,7 +35,7 @@ class TwistMoe(BaseAnime): @classmethod def search(self, query): - r = requests.get('https://twist.moe', **util.get_requests_options()) + r = session.get('https://twist.moe') soup = BeautifulSoup(r.text, 'html.parser') all_anime = soup.select_one('nav.series').select('li') animes = [] @@ -50,12 +51,11 @@ class TwistMoe(BaseAnime): def get_data(self): anime_name = self.url.split('/a/')[-1].split('/')[0] url = self._api_url.format(anime_name) - episodes = requests.get( + episodes = session.get( url, headers={ 'x-access-token': '1rj2vRtegS8Y60B3w3qNZm5T2Q0TN2NR' - }, - **util.get_requests_options() + } ) episodes = episodes.json() self.title = anime_name diff --git a/anime_downloader/util.py b/anime_downloader/util.py index 934e195..b786bec 100644 --- a/anime_downloader/util.py +++ b/anime_downloader/util.py @@ -12,6 +12,7 @@ import time import ast from copy import deepcopy +from anime_downloader.session import session from anime_downloader.sites import get_anime_class from anime_downloader.const import desktop_headers @@ -23,10 +24,6 @@ def set_default_requests_options(**options): _requests_options.update(options) -def get_requests_options(): - return deepcopy(_requests_options) - - def setup_logger(log_level): if log_level == 'DEBUG': format = '%(levelname)s %(name)s: %(message)s' @@ -122,7 +119,7 @@ def print_info(version): def get_json(url, params=None): logging.debug('API call URL: {} with params {!r}'.format(url, params)) - res = requests.get(url, headers=desktop_headers, params=params, **get_requests_options()) + res = session.get(url, headers=desktop_headers, params=params) logging.debug('URL: {}'.format(res.url)) data = res.json() logging.debug('Returned data: {}'.format(data))