Move to session object
parent
6d71d8a899
commit
2af23f7eaf
|
@ -4,6 +4,7 @@ import os
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites import get_anime_class
|
from anime_downloader.sites import get_anime_class
|
||||||
from anime_downloader.players.mpv import mpv
|
from anime_downloader.players.mpv import mpv
|
||||||
from anime_downloader.__version__ import __version__
|
from anime_downloader.__version__ import __version__
|
||||||
|
@ -95,10 +96,7 @@ def dl(ctx, anime_url, episode_range, url, player, skip_download, quality,
|
||||||
cls = get_anime_class(anime_url)
|
cls = get_anime_class(anime_url)
|
||||||
|
|
||||||
disable_ssl = cls and cls.__name__ == 'Masterani' or disable_ssl
|
disable_ssl = cls and cls.__name__ == 'Masterani' or disable_ssl
|
||||||
util.set_default_requests_options(
|
session.verify = not disable_ssl
|
||||||
# Set global options for requests here. For example: timeout=N, proxies={}
|
|
||||||
verify=not disable_ssl
|
|
||||||
)
|
|
||||||
|
|
||||||
if not cls:
|
if not cls:
|
||||||
anime_url = util.search(anime_url, provider)
|
anime_url = util.search(anime_url, provider)
|
||||||
|
|
|
@ -5,6 +5,7 @@ import logging
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
|
from anime_downloader.session import session
|
||||||
|
|
||||||
|
|
||||||
class BaseDownloader:
|
class BaseDownloader:
|
||||||
|
@ -20,7 +21,7 @@ class BaseDownloader:
|
||||||
|
|
||||||
self.chunksize = 16384
|
self.chunksize = 16384
|
||||||
|
|
||||||
r = requests.get(self.url, stream=True, **util.get_requests_options())
|
r = session.get(self.url, stream=True)
|
||||||
|
|
||||||
self.total_size = int(r.headers['Content-length'])
|
self.total_size = int(r.headers['Content-length'])
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
|
|
|
@ -3,6 +3,7 @@ import os
|
||||||
|
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
from anime_downloader.downloader.base_downloader import BaseDownloader
|
from anime_downloader.downloader.base_downloader import BaseDownloader
|
||||||
|
from anime_downloader.session import session
|
||||||
|
|
||||||
|
|
||||||
class HTTPDownloader(BaseDownloader):
|
class HTTPDownloader(BaseDownloader):
|
||||||
|
@ -22,12 +23,11 @@ class HTTPDownloader(BaseDownloader):
|
||||||
with open(self.path, 'w'):
|
with open(self.path, 'w'):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
r = requests.get(self.url, stream=True, **util.get_requests_options())
|
r = session.get(self.url, stream=True)
|
||||||
while self.downloaded < self.total_size:
|
while self.downloaded < self.total_size:
|
||||||
r = requests.get(self.url,
|
r = session.get(self.url,
|
||||||
headers=set_range(range_start, range_end),
|
headers=set_range(range_start, range_end),
|
||||||
stream=True,
|
stream=True)
|
||||||
**util.get_requests_options())
|
|
||||||
if r.status_code == 206:
|
if r.status_code == 206:
|
||||||
with open(self.path, 'ab') as f:
|
with open(self.path, 'ab') as f:
|
||||||
for chunk in r.iter_content(chunk_size=self.chunksize):
|
for chunk in r.iter_content(chunk_size=self.chunksize):
|
||||||
|
@ -43,7 +43,7 @@ class HTTPDownloader(BaseDownloader):
|
||||||
range_end = ''
|
range_end = ''
|
||||||
|
|
||||||
def _non_range_download(self):
|
def _non_range_download(self):
|
||||||
r = requests.get(self.url, stream=True, **util.get_requests_options())
|
r = session.get(self.url, stream=True)
|
||||||
|
|
||||||
if r.status_code == 200:
|
if r.status_code == 200:
|
||||||
with open(self.path, 'wb') as f:
|
with open(self.path, 'wb') as f:
|
||||||
|
|
|
@ -4,12 +4,13 @@ import base64
|
||||||
|
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
from anime_downloader.extractors.base_extractor import BaseExtractor
|
from anime_downloader.extractors.base_extractor import BaseExtractor
|
||||||
|
from anime_downloader.session import session
|
||||||
|
|
||||||
|
|
||||||
class StreamMoe(BaseExtractor):
|
class StreamMoe(BaseExtractor):
|
||||||
def _get_data(self):
|
def _get_data(self):
|
||||||
url = self.url
|
url = self.url
|
||||||
res = requests.get(url, **util.get_requests_options())
|
res = session.get(url)
|
||||||
content_re = re.compile(r"= atob\('(.*?)'\)")
|
content_re = re.compile(r"= atob\('(.*?)'\)")
|
||||||
source_re = re.compile(r'source src="(.*?)"')
|
source_re = re.compile(r'source src="(.*?)"')
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
from anime_downloader.extractors.base_extractor import BaseExtractor
|
from anime_downloader.extractors.base_extractor import BaseExtractor
|
||||||
|
from anime_downloader.session import session
|
||||||
|
|
||||||
|
|
||||||
class MP4Upload(BaseExtractor):
|
class MP4Upload(BaseExtractor):
|
||||||
|
@ -21,7 +22,7 @@ class MP4Upload(BaseExtractor):
|
||||||
r'.*?(www\d+).*?\|video\|(.*?)\|(\d+)\|.*?',
|
r'.*?(www\d+).*?\|video\|(.*?)\|(\d+)\|.*?',
|
||||||
re.DOTALL)
|
re.DOTALL)
|
||||||
|
|
||||||
mp4u_embed = requests.get(self.url, **util.get_requests_options()).text
|
mp4u_embed = session.get(self.url).text
|
||||||
domain, video_id, protocol = source_parts_re.match(mp4u_embed).groups()
|
domain, video_id, protocol = source_parts_re.match(mp4u_embed).groups()
|
||||||
|
|
||||||
logging.debug('Domain: %s, Video ID: %s, Protocol: %s' %
|
logging.debug('Domain: %s, Video ID: %s, Protocol: %s' %
|
||||||
|
@ -29,7 +30,7 @@ class MP4Upload(BaseExtractor):
|
||||||
|
|
||||||
url = self.url.replace('embed-', '')
|
url = self.url.replace('embed-', '')
|
||||||
# Return to non-embed page to collect title
|
# Return to non-embed page to collect title
|
||||||
mp4u_page = BeautifulSoup(requests.get(url, **util.get_requests_options()).text, 'html.parser')
|
mp4u_page = BeautifulSoup(session.get(url).text, 'html.parser')
|
||||||
|
|
||||||
title = mp4u_page.find('span', {'class': 'dfilename'}).text
|
title = mp4u_page.find('span', {'class': 'dfilename'}).text
|
||||||
title = title[:title.rfind('_')][:title.rfind('.')].replace(' ', '_')
|
title = title[:title.rfind('_')][:title.rfind('.')].replace(' ', '_')
|
||||||
|
|
|
@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
from anime_downloader.extractors.base_extractor import BaseExtractor
|
from anime_downloader.extractors.base_extractor import BaseExtractor
|
||||||
|
from anime_downloader.session import session
|
||||||
|
|
||||||
|
|
||||||
class RapidVideo(BaseExtractor):
|
class RapidVideo(BaseExtractor):
|
||||||
|
@ -14,13 +15,13 @@ class RapidVideo(BaseExtractor):
|
||||||
headers = self.headers
|
headers = self.headers
|
||||||
headers['referer'] = url
|
headers['referer'] = url
|
||||||
try:
|
try:
|
||||||
r = requests.get(url, headers=headers, **util.get_requests_options())
|
r = session.get(url, headers=headers)
|
||||||
except:
|
except:
|
||||||
r = requests.post(url, {
|
r = session.post(url, {
|
||||||
'cursor.x': 12,
|
'cursor.x': 12,
|
||||||
'cursor.y': 12,
|
'cursor.y': 12,
|
||||||
'block': 1,
|
'block': 1,
|
||||||
}, headers=headers, **util.get_requests_options())
|
}, headers=headers)
|
||||||
soup = BeautifulSoup(r.text, 'html.parser')
|
soup = BeautifulSoup(r.text, 'html.parser')
|
||||||
|
|
||||||
# TODO: Make these a different function. Can be reused in other classes
|
# TODO: Make these a different function. Can be reused in other classes
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
import requests
|
||||||
|
|
||||||
|
session = requests.Session()
|
|
@ -7,6 +7,7 @@ import logging
|
||||||
import sys
|
import sys
|
||||||
import copy
|
import copy
|
||||||
|
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites.exceptions import AnimeDLError, NotFoundError
|
from anime_downloader.sites.exceptions import AnimeDLError, NotFoundError
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
from anime_downloader.const import desktop_headers
|
from anime_downloader.const import desktop_headers
|
||||||
|
@ -49,7 +50,7 @@ class BaseAnime:
|
||||||
|
|
||||||
def get_data(self):
|
def get_data(self):
|
||||||
self._episode_urls = []
|
self._episode_urls = []
|
||||||
r = requests.get(self.url, headers=desktop_headers, **util.get_requests_options())
|
r = session.get(self.url, headers=desktop_headers)
|
||||||
soup = BeautifulSoup(r.text, 'html.parser')
|
soup = BeautifulSoup(r.text, 'html.parser')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites.anime import BaseAnime, BaseEpisode
|
from anime_downloader.sites.anime import BaseAnime, BaseEpisode
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
|
@ -10,7 +11,7 @@ class GogoanimeEpisode(BaseEpisode):
|
||||||
_base_url = 'https://www2.gogoanime.se'
|
_base_url = 'https://www2.gogoanime.se'
|
||||||
|
|
||||||
def _get_sources(self):
|
def _get_sources(self):
|
||||||
soup = BeautifulSoup(requests.get(self.url, **util.get_requests_options()).text, 'html.parser')
|
soup = BeautifulSoup(session.get(self.url).text, 'html.parser')
|
||||||
url = 'https:'+soup.select_one('li.anime a').get('data-video')
|
url = 'https:'+soup.select_one('li.anime a').get('data-video')
|
||||||
|
|
||||||
res = requests.get(url)
|
res = requests.get(url)
|
||||||
|
@ -35,7 +36,7 @@ class GogoAnime(BaseAnime):
|
||||||
'id': anime_id,
|
'id': anime_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
res = requests.get(self._episode_list_url, params=params, **util.get_requests_options())
|
res = session.get(self._episode_list_url, params=params)
|
||||||
soup = BeautifulSoup(res.text, 'html.parser')
|
soup = BeautifulSoup(res.text, 'html.parser')
|
||||||
|
|
||||||
epurls = list(
|
epurls = list(
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites.kissanime import KissAnime
|
from anime_downloader.sites.kissanime import KissAnime
|
||||||
from anime_downloader.sites.anime import BaseEpisode, SearchResult
|
from anime_downloader.sites.anime import BaseEpisode, SearchResult
|
||||||
from anime_downloader.sites.exceptions import NotFoundError
|
from anime_downloader.sites.exceptions import NotFoundError
|
||||||
|
@ -25,13 +26,12 @@ class KisscartoonEpisode(BaseEpisode):
|
||||||
}
|
}
|
||||||
headers = desktop_headers
|
headers = desktop_headers
|
||||||
headers['referer'] = self.url
|
headers['referer'] = self.url
|
||||||
res = requests.get(self._episode_list_url,
|
res = session.get(self._episode_list_url, params=params, headers=headers)
|
||||||
params=params, headers=headers, **util.get_requests_options())
|
|
||||||
url = res.json()['value']
|
url = res.json()['value']
|
||||||
|
|
||||||
headers = desktop_headers
|
headers = desktop_headers
|
||||||
headers['referer'] = self.url
|
headers['referer'] = self.url
|
||||||
res = requests.get('https:' + url, headers=headers, **util.get_requests_options())
|
res = session.get('https:' + url, headers=headers)
|
||||||
|
|
||||||
return [(
|
return [(
|
||||||
'no_extractor',
|
'no_extractor',
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites.anime import BaseAnime, BaseEpisode, SearchResult
|
from anime_downloader.sites.anime import BaseAnime, BaseEpisode, SearchResult
|
||||||
from anime_downloader.sites.exceptions import NotFoundError, AnimeDLError
|
from anime_downloader.sites.exceptions import NotFoundError, AnimeDLError
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
|
@ -59,8 +60,7 @@ class NineAnime(BaseAnime):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def search(cls, query):
|
def search(cls, query):
|
||||||
r = requests.get('https://www4.9anime.is/search?',
|
r = session.get('https://www4.9anime.is/search?', params={'keyword': query}, headers=desktop_headers)
|
||||||
params={'keyword': query}, headers=desktop_headers, **util.get_requests_options())
|
|
||||||
|
|
||||||
logging.debug(r.url)
|
logging.debug(r.url)
|
||||||
|
|
||||||
|
@ -104,7 +104,7 @@ class NineAnime(BaseAnime):
|
||||||
params = {}
|
params = {}
|
||||||
params['_'] = int(generate_(params))
|
params['_'] = int(generate_(params))
|
||||||
params['_'] = 648
|
params['_'] = 648
|
||||||
soup = BeautifulSoup(requests.get(api_url, params=params, **util.get_requests_options()).json()['html'], 'html.parser')
|
soup = BeautifulSoup(session.get(api_url, params=params).json()['html'], 'html.parser')
|
||||||
episodes = soup.find('div', {'class': 'server', 'data-name': 33})
|
episodes = soup.find('div', {'class': 'server', 'data-name': 33})
|
||||||
episodes = episodes.find_all('li')
|
episodes = episodes.find_all('li')
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@ from bs4 import BeautifulSoup
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
from anime_downloader import util
|
from anime_downloader import util
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites.anime import BaseAnime, BaseEpisode, SearchResult
|
from anime_downloader.sites.anime import BaseAnime, BaseEpisode, SearchResult
|
||||||
|
|
||||||
|
|
||||||
|
@ -34,7 +35,7 @@ class TwistMoe(BaseAnime):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def search(self, query):
|
def search(self, query):
|
||||||
r = requests.get('https://twist.moe', **util.get_requests_options())
|
r = session.get('https://twist.moe')
|
||||||
soup = BeautifulSoup(r.text, 'html.parser')
|
soup = BeautifulSoup(r.text, 'html.parser')
|
||||||
all_anime = soup.select_one('nav.series').select('li')
|
all_anime = soup.select_one('nav.series').select('li')
|
||||||
animes = []
|
animes = []
|
||||||
|
@ -50,12 +51,11 @@ class TwistMoe(BaseAnime):
|
||||||
def get_data(self):
|
def get_data(self):
|
||||||
anime_name = self.url.split('/a/')[-1].split('/')[0]
|
anime_name = self.url.split('/a/')[-1].split('/')[0]
|
||||||
url = self._api_url.format(anime_name)
|
url = self._api_url.format(anime_name)
|
||||||
episodes = requests.get(
|
episodes = session.get(
|
||||||
url,
|
url,
|
||||||
headers={
|
headers={
|
||||||
'x-access-token': '1rj2vRtegS8Y60B3w3qNZm5T2Q0TN2NR'
|
'x-access-token': '1rj2vRtegS8Y60B3w3qNZm5T2Q0TN2NR'
|
||||||
},
|
}
|
||||||
**util.get_requests_options()
|
|
||||||
)
|
)
|
||||||
episodes = episodes.json()
|
episodes = episodes.json()
|
||||||
self.title = anime_name
|
self.title = anime_name
|
||||||
|
|
|
@ -12,6 +12,7 @@ import time
|
||||||
import ast
|
import ast
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
|
|
||||||
|
from anime_downloader.session import session
|
||||||
from anime_downloader.sites import get_anime_class
|
from anime_downloader.sites import get_anime_class
|
||||||
from anime_downloader.const import desktop_headers
|
from anime_downloader.const import desktop_headers
|
||||||
|
|
||||||
|
@ -23,10 +24,6 @@ def set_default_requests_options(**options):
|
||||||
_requests_options.update(options)
|
_requests_options.update(options)
|
||||||
|
|
||||||
|
|
||||||
def get_requests_options():
|
|
||||||
return deepcopy(_requests_options)
|
|
||||||
|
|
||||||
|
|
||||||
def setup_logger(log_level):
|
def setup_logger(log_level):
|
||||||
if log_level == 'DEBUG':
|
if log_level == 'DEBUG':
|
||||||
format = '%(levelname)s %(name)s: %(message)s'
|
format = '%(levelname)s %(name)s: %(message)s'
|
||||||
|
@ -122,7 +119,7 @@ def print_info(version):
|
||||||
|
|
||||||
def get_json(url, params=None):
|
def get_json(url, params=None):
|
||||||
logging.debug('API call URL: {} with params {!r}'.format(url, params))
|
logging.debug('API call URL: {} with params {!r}'.format(url, params))
|
||||||
res = requests.get(url, headers=desktop_headers, params=params, **get_requests_options())
|
res = session.get(url, headers=desktop_headers, params=params)
|
||||||
logging.debug('URL: {}'.format(res.url))
|
logging.debug('URL: {}'.format(res.url))
|
||||||
data = res.json()
|
data = res.json()
|
||||||
logging.debug('Returned data: {}'.format(data))
|
logging.debug('Returned data: {}'.format(data))
|
||||||
|
|
Loading…
Reference in New Issue