From 07214f40073e2e5a0af82eb019d37da362d31440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 26 Aug 2017 21:47:45 +0200 Subject: [PATCH] [booru] place subcategories into base classes --- gallery_dl/extractor/3dbooru.py | 3 --- gallery_dl/extractor/booru.py | 5 +++++ gallery_dl/extractor/danbooru.py | 4 ---- gallery_dl/extractor/e621.py | 5 +---- gallery_dl/extractor/gelbooru.py | 10 ---------- gallery_dl/extractor/konachan.py | 5 +---- gallery_dl/extractor/rule34.py | 2 -- gallery_dl/extractor/safebooru.py | 2 -- gallery_dl/extractor/yandere.py | 6 +----- test/test_extractors.py | 1 - 10 files changed, 8 insertions(+), 35 deletions(-) diff --git a/gallery_dl/extractor/3dbooru.py b/gallery_dl/extractor/3dbooru.py index 1b25d266..67f10bc8 100644 --- a/gallery_dl/extractor/3dbooru.py +++ b/gallery_dl/extractor/3dbooru.py @@ -24,7 +24,6 @@ class ThreedeebooruExtractor(booru.JSONBooruExtractor): class ThreedeebooruTagExtractor(ThreedeebooruExtractor, booru.BooruTagExtractor): """Extractor for images from behoimi.org based on search-tags""" - subcategory = "tag" pattern = [r"(?:https?://)?(?:www\.)?behoimi\.org/post" r"(?:/(?:index)?)?\?tags=([^&]+)"] test = [("http://behoimi.org/post?tags=himekawa_azuru dress", { @@ -36,7 +35,6 @@ class ThreedeebooruTagExtractor(ThreedeebooruExtractor, class ThreedeebooruPoolExtractor(ThreedeebooruExtractor, booru.BooruPoolExtractor): """Extractor for image-pools from behoimi.org""" - subcategory = "pool" pattern = [r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(\d+)"] test = [("http://behoimi.org/pool/show/27", { "url": "da75d2d1475449d5ef0c266cb612683b110a30f2", @@ -47,7 +45,6 @@ class ThreedeebooruPoolExtractor(ThreedeebooruExtractor, class ThreedeebooruPostExtractor(ThreedeebooruExtractor, booru.BooruPostExtractor): """Extractor for single images from behoimi.org""" - subcategory = "post" pattern = [r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(\d+)"] test = [("http://behoimi.org/post/show/140852", { "url": "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", diff --git a/gallery_dl/extractor/booru.py b/gallery_dl/extractor/booru.py index 96626be1..c9433a01 100644 --- a/gallery_dl/extractor/booru.py +++ b/gallery_dl/extractor/booru.py @@ -107,6 +107,7 @@ class XMLBooruExtractor(BooruExtractor): class BooruTagExtractor(BooruExtractor): """Extractor for images based on search-tags""" + subcategory = "tag" directory_fmt = ["{category}", "{tags}"] def __init__(self, match): @@ -120,6 +121,7 @@ class BooruTagExtractor(BooruExtractor): class BooruPoolExtractor(BooruExtractor): """Extractor for image-pools""" + subcategory = "pool" directory_fmt = ["{category}", "pool", "{pool}"] def __init__(self, match): @@ -133,6 +135,8 @@ class BooruPoolExtractor(BooruExtractor): class BooruPostExtractor(BooruExtractor): """Extractor for single images""" + subcategory = "post" + def __init__(self, match): BooruExtractor.__init__(self) self.post = match.group(1) @@ -141,6 +145,7 @@ class BooruPostExtractor(BooruExtractor): class BooruPopularExtractor(BooruExtractor): """Extractor for popular images""" + subcategory = "popular" directory_fmt = ["{category}", "popular", "{scale}", "{date}"] def __init__(self, match): diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index e73629b0..a051f2c2 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -19,7 +19,6 @@ class DanbooruExtractor(booru.JSONBooruExtractor): class DanbooruTagExtractor(DanbooruExtractor, booru.BooruTagExtractor): """Extractor for images from danbooru based on search-tags""" - subcategory = "tag" pattern = [r"(?:https?://)?danbooru\.donmai\.us/posts" r"\?(?:utf8=%E2%9C%93&)?tags=([^&]+)"] test = [("https://danbooru.donmai.us/posts?tags=bonocho", { @@ -29,7 +28,6 @@ class DanbooruTagExtractor(DanbooruExtractor, booru.BooruTagExtractor): class DanbooruPoolExtractor(DanbooruExtractor, booru.BooruPoolExtractor): """Extractor for image-pools from danbooru""" - subcategory = "pool" pattern = [r"(?:https?://)?danbooru\.donmai\.us/pools/(\d+)"] test = [("https://danbooru.donmai.us/pools/7659", { "content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", @@ -38,7 +36,6 @@ class DanbooruPoolExtractor(DanbooruExtractor, booru.BooruPoolExtractor): class DanbooruPostExtractor(DanbooruExtractor, booru.BooruPostExtractor): """Extractor for single images from danbooru""" - subcategory = "post" pattern = [r"(?:https?://)?danbooru\.donmai\.us/posts/(\d+)"] test = [("https://danbooru.donmai.us/posts/294929", { "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", @@ -47,7 +44,6 @@ class DanbooruPostExtractor(DanbooruExtractor, booru.BooruPostExtractor): class DanbooruPopularExtractor(DanbooruExtractor, booru.BooruPopularExtractor): """Extractor for popular images from danbooru""" - subcategory = "popular" pattern = [r"(?:https?://)?danbooru\.donmai\.us/" r"explore/posts/popular()(?:\?([^#]*))?"] test = [ diff --git a/gallery_dl/extractor/e621.py b/gallery_dl/extractor/e621.py index 71914cf4..ef0e5622 100644 --- a/gallery_dl/extractor/e621.py +++ b/gallery_dl/extractor/e621.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 Mike Fährmann +# Copyright 2014-2017 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,7 +19,6 @@ class E621Extractor(booru.JSONBooruExtractor): class E621TagExtractor(E621Extractor, booru.BooruTagExtractor): """Extractor for images from e621.net based on search-tags""" - subcategory = "tag" pattern = [ r"(?:https?://)?(?:www\.)?e621\.net/post/index/\d+/([^?]+)", r"(?:https?://)?(?:www\.)?e621\.net/post\?tags=([^&]+)", @@ -32,7 +31,6 @@ class E621TagExtractor(E621Extractor, booru.BooruTagExtractor): class E621PoolExtractor(E621Extractor, booru.BooruPoolExtractor): """Extractor for image-pools from e621.net""" - subcategory = "pool" pattern = [r"(?:https?://)?(?:www\.)?e621\.net/pool/show/(\d+)"] test = [("https://e621.net/pool/show/73", { "url": "842f2fb065c7c339486a9b1d689020b8569888ed", @@ -42,7 +40,6 @@ class E621PoolExtractor(E621Extractor, booru.BooruPoolExtractor): class E621PostExtractor(E621Extractor, booru.BooruPostExtractor): """Extractor for single images from e621.net""" - subcategory = "post" pattern = [r"(?:https?://)?(?:www\.)?e621\.net/post/show/(\d+)"] test = [("https://e621.net/post/show/535", { "url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index 859d926a..6d83bc72 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -20,18 +20,10 @@ class GelbooruExtractor(booru.XMLBooruExtractor): def setup(self): self.params.update({"page": "dapi", "s": "post", "q": "index"}) - try: - cookies = self.config("cookies") - self.session.cookies.update({ - key: str(value) for key, value in cookies.items() - }) - except AttributeError: - pass class GelbooruTagExtractor(GelbooruExtractor, booru.BooruTagExtractor): """Extractor for images from gelbooru.com based on search-tags""" - subcategory = "tag" pattern = [r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?" r"\?page=post&s=list&tags=([^&]+)"] test = [("http://gelbooru.com/index.php?page=post&s=list&tags=bonocho", { @@ -42,14 +34,12 @@ class GelbooruTagExtractor(GelbooruExtractor, booru.BooruTagExtractor): # TODO: find out how to access pools via gelbooru-api # class GelbooruPoolExtractor(GelbooruExtractor, booru.BooruPoolExtractor): # """Extractor for image-pools from gelbooru.com""" - # subcategory = "pool" # pattern = [r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?" # r"\?page=pool&s=show&id=(\d+)"] class GelbooruPostExtractor(GelbooruExtractor, booru.BooruPostExtractor): """Extractor for single images from gelbooru.com""" - subcategory = "post" pattern = [r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?" r"\?page=post&s=view&id=(\d+)"] test = [("http://gelbooru.com/index.php?page=post&s=view&id=313638", { diff --git a/gallery_dl/extractor/konachan.py b/gallery_dl/extractor/konachan.py index 8eda1095..7859be2a 100644 --- a/gallery_dl/extractor/konachan.py +++ b/gallery_dl/extractor/konachan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015 Mike Fährmann +# Copyright 2015-2017 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,7 +19,6 @@ class KonachanExtractor(booru.JSONBooruExtractor): class KonachanTagExtractor(KonachanExtractor, booru.BooruTagExtractor): """Extractor for images from konachan.com based on search-tags""" - subcategory = "tag" pattern = [r"(?:https?://)?(?:www\.)?konachan\.com/post\?tags=([^&]+)"] test = [("http://konachan.com/post?tags=patata", { "content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", @@ -28,7 +27,6 @@ class KonachanTagExtractor(KonachanExtractor, booru.BooruTagExtractor): class KonachanPoolExtractor(KonachanExtractor, booru.BooruPoolExtractor): """Extractor for image-pools from konachan.com""" - subcategory = "pool" pattern = [r"(?:https?://)?(?:www\.)?konachan\.com/pool/show/(\d+)"] test = [("http://konachan.com/pool/show/95", { "content": "cf0546e38a93c2c510a478f8744e60687b7a8426", @@ -37,7 +35,6 @@ class KonachanPoolExtractor(KonachanExtractor, booru.BooruPoolExtractor): class KonachanPostExtractor(KonachanExtractor, booru.BooruPostExtractor): """Extractor for single images from konachan.com""" - subcategory = "post" pattern = [r"(?:https?://)?(?:www\.)?konachan\.com/post/show/(\d+)"] test = [("http://konachan.com/post/show/205189", { "content": "674e75a753df82f5ad80803f575818b8e46e4b65", diff --git a/gallery_dl/extractor/rule34.py b/gallery_dl/extractor/rule34.py index 158bc04b..030e9201 100644 --- a/gallery_dl/extractor/rule34.py +++ b/gallery_dl/extractor/rule34.py @@ -24,7 +24,6 @@ class Rule34Extractor(booru.XMLBooruExtractor): class Rule34TagExtractor(Rule34Extractor, booru.BooruTagExtractor): """Extractor for images from rule34.xxx based on search-tags""" - subcategory = "tag" pattern = [(r"(?:https?://)?(?:www\.)?rule34\.xxx/(?:index\.php)?" r"\?page=post&s=list&tags=([^&]+)")] test = [("http://rule34.xxx/index.php?page=post&s=list&tags=danraku", { @@ -35,7 +34,6 @@ class Rule34TagExtractor(Rule34Extractor, booru.BooruTagExtractor): class Rule34PostExtractor(Rule34Extractor, booru.BooruPostExtractor): """Extractor for single images from rule34.xxx""" - subcategory = "post" pattern = [(r"(?:https?://)?(?:www\.)?rule34\.xxx/(?:index\.php)?" r"\?page=post&s=view&id=(\d+)")] test = [("http://rule34.xxx/index.php?page=post&s=view&id=1974854", { diff --git a/gallery_dl/extractor/safebooru.py b/gallery_dl/extractor/safebooru.py index 073ac312..72068e7f 100644 --- a/gallery_dl/extractor/safebooru.py +++ b/gallery_dl/extractor/safebooru.py @@ -24,7 +24,6 @@ class SafebooruExtractor(booru.XMLBooruExtractor): class SafebooruTagExtractor(SafebooruExtractor, booru.BooruTagExtractor): """Extractor for images from safebooru.org based on search-tags""" - subcategory = "tag" pattern = [(r"(?:https?://)?(?:www\.)?safebooru\.org/(?:index\.php)?" r"\?page=post&s=list&tags=([^&]+)")] test = [("http://safebooru.org/index.php?page=post&s=list&tags=bonocho", { @@ -35,7 +34,6 @@ class SafebooruTagExtractor(SafebooruExtractor, booru.BooruTagExtractor): class SafebooruPostExtractor(SafebooruExtractor, booru.BooruPostExtractor): """Extractor for single images from safebooru.org""" - subcategory = "post" pattern = [(r"(?:https?://)?(?:www\.)?safebooru\.org/(?:index\.php)?" r"\?page=post&s=view&id=(\d+)")] test = [("http://safebooru.org/index.php?page=post&s=view&id=1169132", { diff --git a/gallery_dl/extractor/yandere.py b/gallery_dl/extractor/yandere.py index ac1290be..6ec2f236 100644 --- a/gallery_dl/extractor/yandere.py +++ b/gallery_dl/extractor/yandere.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015 Mike Fährmann +# Copyright 2015-2017 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,7 +19,6 @@ class YandereExtractor(booru.JSONBooruExtractor): class YandereTagExtractor(YandereExtractor, booru.BooruTagExtractor): """Extractor for images from yande.re based on search-tags""" - subcategory = "tag" pattern = [r"(?:https?://)?(?:www\.)?yande\.re/post\?tags=([^&]+)"] test = [("https://yande.re/post?tags=ouzoku armor", { "content": "59201811c728096b2d95ce6896fd0009235fe683", @@ -28,7 +27,6 @@ class YandereTagExtractor(YandereExtractor, booru.BooruTagExtractor): class YanderePoolExtractor(YandereExtractor, booru.BooruPoolExtractor): """Extractor for image-pools from yande.re""" - subcategory = "pool" pattern = [r"(?:https?://)?(?:www\.)?yande\.re/pool/show/(\d+)"] test = [("https://yande.re/pool/show/318", { "content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", @@ -37,7 +35,6 @@ class YanderePoolExtractor(YandereExtractor, booru.BooruPoolExtractor): class YanderePostExtractor(YandereExtractor, booru.BooruPostExtractor): """Extractor for single images from yande.re""" - subcategory = "post" pattern = [r"(?:https?://)?(?:www\.)?yande\.re/post/show/(\d+)"] test = [("https://yande.re/post/show/51824", { "content": "59201811c728096b2d95ce6896fd0009235fe683", @@ -46,7 +43,6 @@ class YanderePostExtractor(YandereExtractor, booru.BooruPostExtractor): class YanderePopularExtractor(YandereExtractor, booru.BooruPopularExtractor): """Extractor for popular images from yande.re""" - subcategory = "popular" pattern = [r"(?:https?://)?(?:www\.)?yande\.re/post/popular_" r"(by_(?:day|week|month)|recent)(?:\?([^#]*))?"] test = [ diff --git a/test/test_extractors.py b/test/test_extractors.py index d31d01f7..19cba002 100644 --- a/test/test_extractors.py +++ b/test/test_extractors.py @@ -63,7 +63,6 @@ skip = [ "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie", "archivedmoe", "archiveofsins", "thebarchive", # temporary issues - "mangapark", "danbooru", "yandere", "pixiv", ] # enable selective testing for direct calls if __name__ == '__main__' and len(sys.argv) > 1: