diff --git a/anime_downloader/extractors/base_extractor.py b/anime_downloader/extractors/base_extractor.py index 429505a..6888911 100644 --- a/anime_downloader/extractors/base_extractor.py +++ b/anime_downloader/extractors/base_extractor.py @@ -22,6 +22,9 @@ class BaseExtractor: @property def stream_url(self): + """ + URL of the video stream. + """ if not self._stream_url: self.get_data() diff --git a/anime_downloader/sites/anime.py b/anime_downloader/sites/anime.py index 6958d6a..5fde4e1 100644 --- a/anime_downloader/sites/anime.py +++ b/anime_downloader/sites/anime.py @@ -252,8 +252,10 @@ class AnimeEpisode: Title of the anime meta: dict metadata about the anime. [Can be empty] - QUALITIES: list - Possible qualities for the site + ep_no: string + Episode number/title of the episode + pretty_title: string + Pretty title of episode in format - """ QUALITIES = [] title = '' @@ -309,6 +311,14 @@ class AnimeEpisode: return Config['siteconfig'][self.sitename] def source(self, index=0): + """ + Get the source for episode + + Returns + ------- + `anime_downloader.extractors.base_extractor.BaseExtractor` + Extractor depending on the source. + """ if not self._sources: self.get_data() try: @@ -330,6 +340,18 @@ class AnimeEpisode: def download(self, force=False, path=None, format='{anime_title}_{ep_no}', range_size=None): + """ + Downloads episode. This might be removed in a future release. + + Parameters + ---------- + force: bool + Whether to force download or not. + path: string + Path to the directory/file where the file should be downloaded to. + format: string + The format of the filename if not provided. + """ # TODO: Remove this shit logger.info('Downloading {}'.format(self.pretty_title)) if format: diff --git a/anime_downloader/sites/animeflv.py b/anime_downloader/sites/animeflv.py index 523df89..ab088c8 100644 --- a/anime_downloader/sites/animeflv.py +++ b/anime_downloader/sites/animeflv.py @@ -50,6 +50,10 @@ class Animeflv(Anime, sitename='animeflv'): ] return links[::-1] + def _scrape_metadata(self): + soup = helpers.soupify(helpers.get(self.url).text) + self.title = soup.select_one('h2.Title').text + class AnimeflvEpisode(AnimeEpisode, sitename='animeflv'): SERVERS = [ diff --git a/anime_downloader/sites/helpers/request.py b/anime_downloader/sites/helpers/request.py index 88ea804..0d342be 100644 --- a/anime_downloader/sites/helpers/request.py +++ b/anime_downloader/sites/helpers/request.py @@ -22,6 +22,7 @@ req_session = session.get_session() cf_session = cfscrape.create_scraper(sess=req_session) default_headers = get_random_header() temp_dir = tempfile.mkdtemp(prefix='animedl') +logger.debug(f"HTML file temp_dir: {temp_dir}") def setup(func): @@ -136,6 +137,7 @@ def soupify(res): def _log_response_body(res): import json + import pathlib file = tempfile.mktemp(dir=temp_dir) logger.debug(file) with open(file, 'w') as f: @@ -151,7 +153,7 @@ def _log_response_body(res): data.append({ 'method': res.request.method, 'url': res.url, - 'file': '/' + file.split('/')[-1], + 'file': pathlib.Path(file).name, }) with open(data_file, 'w') as f: json.dump(data, f) diff --git a/anime_downloader/sites/init.py b/anime_downloader/sites/init.py index 01a8e9f..32406be 100644 --- a/anime_downloader/sites/init.py +++ b/anime_downloader/sites/init.py @@ -15,6 +15,20 @@ ALL_ANIME_SITES = [ def get_anime_class(url): + """ + Get anime class corresposing to url or name. + See :py:data:`anime_downloader.sites.ALL_ANIME_SITES` to get the possible anime sites. + + Parameters + ---------- + url: string + URL of the anime. + + Returns + ------- + :py:class:`anime_downloader.sites.anime.Anime` + Concrete implementation of :py:class:`anime_downloader.sites.anime.Anime` + """ for site in ALL_ANIME_SITES: if site[1] in url: try: diff --git a/anime_downloader/sites/itsaturday.py b/anime_downloader/sites/itsaturday.py index e61d1cf..4ba6bcb 100644 --- a/anime_downloader/sites/itsaturday.py +++ b/anime_downloader/sites/itsaturday.py @@ -3,7 +3,7 @@ from anime_downloader.sites import helpers class Itsaturday(Anime, sitename='itsaturday'): sitename = 'itsaturday' - DOMAIN = 'http://itsaturday.com' + DOMAIN = 'http://www.itsaturday.com' @classmethod def search(cls, query): @@ -22,6 +22,10 @@ class Itsaturday(Anime, sitename='itsaturday'): ] return ret + def _scrape_metadata(self): + soup = helpers.soupify(helpers.get(self.url)) + self.title = soup.select_one('h1.h3').text + class ItsaturdayEpisode(AnimeEpisode, sitename='itsaturday'): def _get_sources(self): diff --git a/docs/api/base_classes.rst b/docs/api/base_classes.rst index 65bc58d..23c7fd2 100644 --- a/docs/api/base_classes.rst +++ b/docs/api/base_classes.rst @@ -3,7 +3,14 @@ Base classes .. automodule:: anime_downloader.sites.anime + +.. autofunction:: anime_downloader.sites.init.get_anime_class +.. autodata:: anime_downloader.commands.dl.sitenames + .. autoclass:: anime_downloader.sites.anime.Anime :members: search, get_data, _scrape_episodes, _scrape_metadata +.. autoclass:: anime_downloader.sites.anime.AnimeEpisode + :members: source, download + .. autoclass:: anime_downloader.sites.anime.SearchResult diff --git a/docs/index.rst b/docs/index.rst index 707c764..d0f9297 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,6 +42,7 @@ NOTE: To download from sites marked `[cloudflare]`, anime-downloader has to be i usage/watch usage/config usage/sites + usage/api advanced/custom_site api/base_classes.rst api/helper_functions.rst diff --git a/docs/usage/api.rst b/docs/usage/api.rst new file mode 100644 index 0000000..5f9e556 --- /dev/null +++ b/docs/usage/api.rst @@ -0,0 +1,30 @@ +Library usage +============= + +Anime downloader can be used as a library too. + + +The following code searches for 'one punch' from twist.moe + +:py:func:`~anime_downloader.sites.init.get_anime_class` can be used to import specific sites using the url one of :py:data:`~anime_downloader.commands.dl.sitenames`. + +.. code:: python + + from anime_downloader.sites import get_anime_class + + Twist = get_anime_class('twist.moe') + search = Twist.search('one punch') + print(search[0].title) + + # You can directly import twist too + from anime_downloader.sites.twistmoe import TwistMoe + anime = TwistMoe(search[0].url) + print(anime) + print(len(anime)) + + # Get first episodes url + print(anime[0].source().stream_url) + + +In the above example `TwistMoe` is a concrete implementation of :py:class:`anime_downloader.sites.anime.Anime`. +Search results is list of :py:class:`anime_downloader.sites.anime.SearchResult`. diff --git a/docs/usage/config.rst b/docs/usage/config.rst index c153b75..9601e1c 100644 --- a/docs/usage/config.rst +++ b/docs/usage/config.rst @@ -16,22 +16,41 @@ The default config file is given below. .. code:: json - { - "dl": { - "download_dir": ".", - "force_download": false, - "file_format": "{anime_title}/{anime_title}_{ep_no}", - "log_level": "INFO", - "player": null, - "quality": "720p", - "skip_download": false, - "url": false - }, - "watch": { - "log_level": "INFO", - "quality": "720p" - } - } + { + "dl": { + "download_dir": ".", + "external_downloader": "{aria2}", + "fallback_qualities": [ + "720p", + "480p", + "360p" + ], + "file_format": "{anime_title}/{anime_title}_{ep_no}", + "force_download": false, + "player": null, + "provider": "9anime", + "quality": "720p", + "skip_download": false, + "url": false + }, + "siteconfig": { + "animeflv": { + "server": "natsuki", + "version": "subbed" + }, + "anistream.xyz": { + "version": "subbed" + }, + "nineanime": { + "server": "mp4upload" + } + }, + "watch": { + "log_level": "INFO", + "provider": "9anime", + "quality": "720p" + } + } .. note:: - For the key ``file_format``, you can set ``anime_title``\ (which refers to the title of the anime) and ``ep_no`` which is the number of the epiosde. diff --git a/docs/usage/installation.rst b/docs/usage/installation.rst index f447fd5..2f269fa 100644 --- a/docs/usage/installation.rst +++ b/docs/usage/installation.rst @@ -5,30 +5,6 @@ Installation The following are extended installation instructions for Windows and Linux users. (For windows choco users, scroll to last) -Windows -~~~~~~~ - -This tool requires python >= 3.3. - -- Go to python windows `downloads section`_. Choose the option - ``Latest python 3 release``. *Note, not python 2 but python 3* -- Download and install it like any other application. **Don’t forget to - select ‘Add Python to PATH’ in the installation screen** -- After installation open a command prompt and type in the following - command. - -:: - - pip install anime-downloader[cloudflare] - -- Enjoy downloading anime. - -NOTE: If you want to use ``watch`` you have to do some more steps. -(Trust me, it is work the additional steps) - Download mpv from `here`_. -- Extract the zip and place ``mpv.exe`` in the folder your command -prompt opens. (The path you see when you open command prompt). - Enjoy. -Read documentation for watch to know more. - Windows via ``choco`` ~~~~~~~~~~~~~~~~~~~~~ @@ -37,17 +13,26 @@ Windows via ``choco`` .. note:: make sure you are running the Command Prompt in "Run as Adminstrator" mode +- Install `Chocolatey`_ Package manager. + - Using the Chocolatey Package Manager:: choco install -y git mpv python3 aria2 nodejs - Once these are installed:: - pip3 install -U git+https://github.com/vn-ki/anime-downloader.git#egg=anime-downloader[cloudflare] + pip3 install -U git+https://github.com/vn-ki/anime-downloader.git - then the commands to view a show would be:: anime watch --provider *Insert provider name* --new +Mac +~~~ + +Anime downloader is avaible from brew.:: + + brew install anime-downloader + Linux ~~~~~ @@ -56,51 +41,21 @@ If you are using linux, you most probably already have python installed. Type ``pip --version`` into your terminal. If it says python2, replace all the following ``pip`` with ``pip3``. -- You only need one command. +- Install aria2. -:: +- Install anime-downloader :: - pip install anime-downloader[cloudflare] + pip3 install anime-downloader + +- To install master branch:: + + pip3 install -U git+https://github.com/vn-ki/anime-downloader.git - Enjoy. -- You can download mpv with your package manager. You can follow `this - guide`_, to install it on ubuntu. -Windows via ``choco`` (Method 2) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Contributed by @1ijack - -- Using the `Chocolatey`_ Package manager (via Windows Nuget) to - - - Install these pre-reqs: `git`_ `python3`_ `aria2`_ `mpv`_ - - Refresh/Update environment changes (choco ``refreshenv`` command) - - clone this repository to - ``[C:]\Users\[USERNAME]\anime-downloader`` - - setup/build anime-downloader from - ``"%userprofile%\anime-downloader"`` - - run ``anime watch`` command - -- the ``choco install`` command SHOULD be run with an administrator - privledged ``cmd.exe`` console: - - - ``[WindowsKey] > "cmd.exe" > [Shift]+[Ctrl]+[Enter] > (Click "Yes")`` - - :: - - choco install -y git python3 aria2 mpv - -- the rest of the commands SHOULD be run in an un-privledged/normal - ``cmd.exe`` console: - - - ``[WindowsKey] > "cmd.exe" > [Enter]`` \``\` refreshenv git clone - https://github.com/vn-ki/anime-downloader - “%userprofile%:raw-latex:`\anime`-downloader” cd /d " - .. _downloads section: https://www.python.org/downloads/windows/ .. _here: https://mpv.srsfckn.biz/ -.. _this guide: http://ubuntuhandbook.org/index.php/2017/12/install-mpv-0-28-0-in-ubuntu-18-04-16-04/ .. _Chocolatey: https://chocolatey.org/install .. _git: https://chocolatey.org/packages/git .. _python3: https://chocolatey.org/packages/python3 diff --git a/docs/usage/sites.rst b/docs/usage/sites.rst index 7c00fbe..55ae475 100644 --- a/docs/usage/sites.rst +++ b/docs/usage/sites.rst @@ -1,6 +1,8 @@ Site Configs ------------ +The following are the sites with configs. These can be configured using the config file. The keys and corresponding possible values are mentioned below. + .. autoclass:: anime_downloader.sites.animeflv.Animeflv .. autoclass:: anime_downloader.sites.anistream.Anistream .. autoclass:: anime_downloader.sites.nineanime.NineAnime diff --git a/tests/sites/site.py b/tests/sites/site.py deleted file mode 100644 index 84d1fa0..0000000 --- a/tests/sites/site.py +++ /dev/null @@ -1,29 +0,0 @@ -from abc import ABCMeta, abstractmethod, abstractproperty -import httpretty -import json - - -class MockSite(metaclass=ABCMeta): - def __init__(self): - self.setup_httpretty() - - def setup_httpretty(self): - data_file = 'webpages/data.json' - data = None - with open(data_file) as f: - data = json.load(f) - - for obj in data: - method = httpretty.POST - if obj['method'] == 'GET': - method = httpretty.GET - with open('webpages' + obj['file']) as f: - httpretty.register_uri( - method, - obj['url'], - f.read(), - ) - - @abstractproperty - def url(self): - raise NotImplementedError diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_extractors.py b/tests/test_extractors.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_itsaturday.py b/tests/test_itsaturday.py new file mode 100644 index 0000000..456aa5a --- /dev/null +++ b/tests/test_itsaturday.py @@ -0,0 +1,28 @@ +import pytest + +from anime_downloader.sites.itsaturday import Itsaturday +from test_sites.site import configure_httpretty + + +@pytest.fixture +def anime(): + return Itsaturday('http://www.itsaturday.com/Star-Wars-The-Clone-Wars-2008---2015-Full-Episodes') + +configure_httpretty('itsaturday') + +def test_search(): + ret = Itsaturday.search('star wars') + assert len(ret) == 16 + assert ret[0].title == 'Star Wars: The Clone Wars (2008 - 2015)' + + +def test_title(anime): + assert anime.title == 'Star Wars: The Clone Wars ' + + +def test_length(anime): + assert len(anime) == 115 + + +def test_streamurl(anime): + assert anime[0].source().stream_url == 'http://www.itsaturday.com/watch.php?url=https://2.bp.blogspot.com/STnnqTh7VY1AKmgvX3DlY_ofRpg26WwLW8a85YP4AihSCrianZGNVtzwIi0cPXiw26jd1yoDfU3XMrWQ7Fq4A99KzeJpaBE8SMP16an1E4NsgvpF_mZlpGyfhOuqTCU3Ux0DRnw8=m18&extra=.mp4' diff --git a/tests/sites/nineanime/nineanime.py b/tests/test_sites/nineanime/nineanime.py similarity index 100% rename from tests/sites/nineanime/nineanime.py rename to tests/test_sites/nineanime/nineanime.py diff --git a/tests/sites/nineanime/webpages/naruto.xx8z b/tests/test_sites/nineanime/webpages/naruto.xx8z similarity index 100% rename from tests/sites/nineanime/webpages/naruto.xx8z rename to tests/test_sites/nineanime/webpages/naruto.xx8z diff --git a/tests/test_sites/site.py b/tests/test_sites/site.py new file mode 100644 index 0000000..2878b85 --- /dev/null +++ b/tests/test_sites/site.py @@ -0,0 +1,25 @@ +from abc import ABCMeta, abstractmethod, abstractproperty +import httpretty +import json +from pathlib import Path + + +def configure_httpretty(sitedir): + httpretty.enable() + dir = Path(f"tests/test_sites/test_{sitedir}/webpages/") + data_file = dir / 'data.json' + + data = None + with open(data_file) as f: + data = json.load(f) + + for obj in data: + method = httpretty.POST + if obj['method'] == 'GET': + method = httpretty.GET + with open(dir / obj['file']) as f: + httpretty.register_uri( + method, + obj['url'], + f.read(), + ) diff --git a/tests/test_sites/test_itsaturday/test_itsaturday.py b/tests/test_sites/test_itsaturday/test_itsaturday.py new file mode 100644 index 0000000..573bafc --- /dev/null +++ b/tests/test_sites/test_itsaturday/test_itsaturday.py @@ -0,0 +1,14 @@ +import pytest + +from anime_downloader.sites.itsaturday import Itsaturday +from test_sites.site import configure_httpretty + +configure_httpretty('itsaturday') + +@pytest.fixture +def anime(): + return Itsaturday('http://www.itsaturday.com/Star-Wars-The-Clone-Wars-2008---2015-Full-Episodes') + + +def test_search(): + ret = Itsaturday.search('star wars') diff --git a/tests/test_sites/test_itsaturday/webpages/data.json b/tests/test_sites/test_itsaturday/webpages/data.json new file mode 100644 index 0000000..b1420d4 --- /dev/null +++ b/tests/test_sites/test_itsaturday/webpages/data.json @@ -0,0 +1 @@ +[{"method": "GET", "url": "http://www.itsaturday.com/search/?q=star+wars", "file": "tmplzg7hcxn"}, {"method": "GET", "url": "http://www.itsaturday.com/Star-Wars-The-Clone-Wars-2008---2015-Full-Episodes", "file": "tmpmft_6nef"}, {"method": "GET", "url": "http://www.itsaturday.com/Star-Wars-The-Clone-Wars-2008---2015-Full-Episodes/Star-Wars-The-Clone-Wars-S1-Ep2---Rising-Malevolence.html", "file": "tmpfxcrdvx3"}] \ No newline at end of file diff --git a/tests/test_sites/test_itsaturday/webpages/tmpfxcrdvx3 b/tests/test_sites/test_itsaturday/webpages/tmpfxcrdvx3 new file mode 100644 index 0000000..095eccb --- /dev/null +++ b/tests/test_sites/test_itsaturday/webpages/tmpfxcrdvx3 @@ -0,0 +1,221 @@ + + + + + + + + + + + + Star Wars: The Clone Wars (2008 - 2015) Full Episodes - Star Wars: The Clone Wars S1 Ep2 – Rising Malevolence | ItSaturday.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+

+

+ + ItSaturday! + + + +

+ + +
+
+
+ + +
+ +
+
+
+ + + + + + +
 fullscreen  next episode
+

  Star Wars: The Clone Wars S1 Ep2 – Rising Malevolence

    Get on high quality DVD for yourself or as a gift
   😺 Get the latest emojis in 1-Click copy-paste
+
+
+ + + + +
+ +Star Wars: The Clone Wars (2008 – 2015) full episodes watch cartoons online.
+Synopsis: The first weekly TV series from Lucasfilm Animation chronicles the adventures of Anakin Skywalker, Yoda, Obi-Wan Kenobi and other popular characters from the “Star Wars” universe during the violent Clone Wars, as dwindling numbers of Jedi knights struggle to restore peace.
+Creator: George Lucas
+Stars: Tom Kane, Dee Bradley Baker, Matt Lanter
+More information: IMDB, Wikipedia

+
+ + +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ + +2023 \ No newline at end of file diff --git a/tests/test_sites/test_itsaturday/webpages/tmplzg7hcxn b/tests/test_sites/test_itsaturday/webpages/tmplzg7hcxn new file mode 100644 index 0000000..318b48b --- /dev/null +++ b/tests/test_sites/test_itsaturday/webpages/tmplzg7hcxn @@ -0,0 +1,151 @@ + + + + + + + + + + + + Watch Cartoons it's Saturday Morning | ItSaturday.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+

+

+ + ItSaturday! + + + +

+ + +
+
+
+ + +
+ +
+
+
+ + + +
+ \ No newline at end of file diff --git a/tests/test_sites/test_itsaturday/webpages/tmpmft_6nef b/tests/test_sites/test_itsaturday/webpages/tmpmft_6nef new file mode 100644 index 0000000..80a1f19 --- /dev/null +++ b/tests/test_sites/test_itsaturday/webpages/tmpmft_6nef @@ -0,0 +1,221 @@ + + + + + + + + + + + + Star Wars: The Clone Wars (2008 - 2015) Full Episodes | ItSaturday.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+

+

+ + ItSaturday! + + + +

+ + +
+
+
+ + +
+ +
+
+
+ + + + + + +
 fullscreen  next episode
+

  Star Wars: The Clone Wars S1 Ep1 – Ambush

    Get on high quality DVD for yourself or as a gift
   🌸 Get the latest emojis in 1-Click copy-paste
+
+
+ + + + +
+ +Star Wars: The Clone Wars (2008 – 2015) full episodes watch cartoons online.
+Synopsis: The first weekly TV series from Lucasfilm Animation chronicles the adventures of Anakin Skywalker, Yoda, Obi-Wan Kenobi and other popular characters from the “Star Wars” universe during the violent Clone Wars, as dwindling numbers of Jedi knights struggle to restore peace.
+Creator: George Lucas
+Stars: Tom Kane, Dee Bradley Baker, Matt Lanter
+More information: IMDB, Wikipedia

+
+ + +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+2023 \ No newline at end of file