mikf/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[sankakucomplex] move article extractor to its own module (#258)

Browse Source
This commit is contained in:
Mike Fährmann 2019-05-27 23:49:23 +02:00
parent 4465a3ea68
commit 74c2415138
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
5 changed files with 91 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/supportedsites.rst
View File

@ -86,7 +86,8 @@ Reddit https://www.reddit.com/ individual Images, Subm
rule #34 https://rule34.paheal.net/ Posts, Tag-Searches
Rule 34 https://rule34.xxx/ Pools, Posts, Tag-Searches
Safebooru https://safebooru.org/ Pools, Posts, Tag-Searches
Sankaku Channel https://chan.sankakucomplex.com/ Articles, Pools, Posts, Tag-Searches Optional
Sankaku Channel https://chan.sankakucomplex.com/ Pools, Posts, Tag-Searches Optional
Sankaku Complex https://www.sankakucomplex.com/ Articles
Sen Manga https://raw.senmanga.com/ Chapters
Sense-Scans http://sensescans.com/reader/ Chapters, Manga
Sex.com https://www.sex.com/ Boards, Pins, Search Results

1
gallery_dl/extractor/__init__.py
View File

@ -78,6 +78,7 @@ modules = [
"rule34",
"safebooru",
"sankaku",
"sankakucomplex",
"seiga",
"senmanga",
"sexcom",

71
gallery_dl/extractor/sankaku.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from https://chan.sankakucomplex.com/"""
"""Extractors for https://chan.sankakucomplex.com/"""
from .common import Extractor, Message, SharedConfigMixin
from .. import text, util, exception
@ -297,72 +297,3 @@ class SankakuPostExtractor(SankakuExtractor):
def get_posts(self):
return (self.post_id,)
class SankakuArticleExtractor(Extractor):
"""Extractor for articles on www.sankakucomplex.com"""
category = "sankaku"
subcategory = "article"
directory_fmt = ("{category}", "Articles", "{date:%Y-%m-%d} {title}")
filename_fmt = "{filename}.{extension}"
archive_fmt = "a_{date:%Y%m%d}_{filename}"
pattern = (r"(?:https?://)?www\.sankakucomplex\.com"
r"/(\d{4}/\d\d/\d\d)/([^/?&#]+)")
test = (
("https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", {
"url": "4a9ecc5ae917fbce469280da5b6a482510cae84d",
"keyword": "4ab96f31df9ee95d0dc6eefc2ca4e508c45c8e00",
}),
("https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", {
"url": "a1e249173fd6c899a8134fcfbd9c925588a63f7c",
"keyword": "a7876de642bf3e68fb4743dcd4d4e8778f2c17ab",
}),
)
root = "https://www.sankakucomplex.com"
def __init__(self, match):
Extractor.__init__(self, match)
self.date, self.title = match.groups()
def items(self):
url = "{}/{}/{}/?pg=X".format(self.root, self.date, self.title)
extr = text.extract_from(self.request(url).text)
data = {
"title" : text.unescape(
extr('"og:title" content="', '"')),
"description": text.unescape(
extr('"og:description" content="', '"')),
"date" : text.parse_datetime(
extr('"og:updated_time" content="', '"')),
}
imgs = self.images(extr)
data["count"] = len(imgs)
data["tags"] = text.split_html(extr('="meta-tags">', '</div>'))[::2]
yield Message.Directory, data
for img in imgs:
img.update(data)
yield Message.Url, img["url"], img
def images(self, extr):
num = 0
imgs = []
urls = set()
orig = re.compile(r"-\d+x\d+\.")
extr('<div class="entry-content">', '')
while True:
url = extr('data-lazy-src="', '"')
if not url:
return imgs
if url in urls:
continue
if url[0] == "/":
url = text.urljoin(self.root, url)
url = orig.sub(".", url)
num += 1
imgs.append(text.nameext_from_url(url, {
"url" : url,
"num" : num,
}))
urls.add(url)

86
gallery_dl/extractor/sankakucomplex.py Normal file
View File

@ -0,0 +1,86 @@
# -*- coding: utf-8 -*-
# Copyright 2019 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://www.sankakucomplex.com/"""
from .common import Extractor, Message
from .. import text
import re
class SankakucomplexExtractor(Extractor):
"""Base class for sankakucomplex extractors"""
category = "sankakucomplex"
root = "https://www.sankakucomplex.com"
def __init__(self, match):
Extractor.__init__(self, match)
self.path = match.group(1)
class SankakucomplexArticleExtractor(SankakucomplexExtractor):
"""Extractor for articles on www.sankakucomplex.com"""
subcategory = "article"
directory_fmt = ("{category}", "{date:%Y-%m-%d} {title}")
filename_fmt = "{filename}.{extension}"
archive_fmt = "{date:%Y%m%d}_{filename}"
pattern = (r"(?:https?://)?www\.sankakucomplex\.com"
r"/(\d{4}/\d\d/\d\d/[^/?&#]+)")
test = (
("https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", {
"url": "4a9ecc5ae917fbce469280da5b6a482510cae84d",
"keyword": "35cd2a0aba712d6b0e27a9fa2a5e823199d10ca0",
}),
("https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", {
"url": "a1e249173fd6c899a8134fcfbd9c925588a63f7c",
"keyword": "8bf60e62fb5e9f2caabb29c16ed58d7e0dcf247f",
}),
)
def items(self):
url = "{}/{}/?pg=X".format(self.root, self.path)
extr = text.extract_from(self.request(url).text)
data = {
"title" : text.unescape(
extr('"og:title" content="', '"')),
"description": text.unescape(
extr('"og:description" content="', '"')),
"date" : text.parse_datetime(
extr('"og:updated_time" content="', '"')),
}
imgs = self.images(extr)
data["count"] = len(imgs)
data["tags"] = text.split_html(extr('="meta-tags">', '</div>'))[::2]
yield Message.Directory, data
for img in imgs:
img.update(data)
yield Message.Url, img["url"], img
def images(self, extr):
num = 0
imgs = []
urls = set()
orig = re.compile(r"-\d+x\d+\.")
extr('<div class="entry-content">', '')
while True:
url = extr('data-lazy-src="', '"')
if not url:
return imgs
if url in urls:
continue
if url[0] == "/":
url = text.urljoin(self.root, url)
url = orig.sub(".", url)
num += 1
imgs.append(text.nameext_from_url(url, {
"url" : url,
"num" : num,
}))
urls.add(url)

1
scripts/supportedsites.py
View File

@ -57,6 +57,7 @@ CATEGORY_MAP = {
"rbt" : "RebeccaBlackTech",
"rule34" : "Rule 34",
"sankaku" : "Sankaku Channel",
"sankakucomplex" : "Sankaku Complex",
"seaotterscans" : "Sea Otter Scans",
"seiga" : "Niconico Seiga",
"senmanga" : "Sen Manga",