mikf/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve supportedsites.rst and build script

Browse Source
This commit is contained in:
Mike Fährmann 2017-04-20 16:56:50 +02:00
parent 13dc5d72bc
commit 9f32cf1f4e
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
3 changed files with 295 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.rst
View File

@ -87,7 +87,7 @@ Supported Sites
* exhentai.org * exhentai.org
* nhentai.net * nhentai.net
* luscious.net * luscious.net
* hentaifoundry.com * hentai-foundry.com
* deviantart.com * deviantart.com
* tumblr.com * tumblr.com
* `Complete List`_ * `Complete List`_

191
scripts/build_supportedsites.py
View File

@ -2,35 +2,186 @@
import sys import sys
import os.path import os.path
import urllib.parse
ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.realpath(ROOTDIR)) sys.path.insert(0, os.path.realpath(ROOTDIR))
import gallery_dl.extractor import gallery_dl.extractor
categories = {} CATEGORY_MAP = {
skip = ["test", "recursive"] "deviantart" : "DeviantArt",
for extr in gallery_dl.extractor.extractors(): "dokireader" : "Doki Reader",
if extr.category in skip: "dynastyscans" : "Dynasty Reader",
continue "e621" : "e621",
"exhentai" : "ExHentai",
"fallenangels" : "Fallen Angels Scans",
"gomanga" : "GoManga",
"hbrowse" : "HBrowse",
"hentai2read" : "Hentai2Read",
"hentaifoundry" : "Hentai Foundry",
"hentaihere" : "HentaiHere",
"hitomi" : "Hitomi.la",
"imagebam" : "ImageBam",
"imagefap" : "ImageFap",
"imgbox" : "imgbox",
"imgchili" : "imgChili",
"imgth" : "imgth",
"imgur" : "imgur",
"jaiminisbox" : "Jaimini's Box",
"kireicake" : "Kirei Cake",
"kisscomic" : "KissComic",
"kissmanga" : "KissManga",
"mangafox" : "Manga Fox",
"mangahere" : "Manga Here",
"mangapark" : "MangaPark",
"mangastream" : "Manga Stream",
"nhentai" : "nhentai",
"nijie" : "nijie",
"powermanga" : "PowerManga",
"readcomiconline": "Read Comic Online",
"rule34" : "Rule 34",
"sankaku" : "Sankaku Channel",
"seaotterscans" : "Sea Otter Scans",
"seiga" : "Niconico Seiga",
"senmanga" : "Sen Manga",
"sensescans" : "Sense-Scans",
"spectrumnexus" : "Spectrum Nexus",
"worldthree" : "World Three",
"yomanga" : "YoManga",
"yonkouprod" : "Yonkou Productions",
}
SUBCATEGORY_MAP = {
"account": "Images from Users",
"gallery": "Galleries",
"image" : "individual Images",
"issue" : "Comic-Issues",
"manga" : "Manga",
"status" : "Images from Statuses",
"tag" : "Tag-Searches",
"user" : "Images from Users",
"work" : "Individual Images",
}
class RstColumn():
def __init__(self, title, data):
self.title = title
self.data = self._transform(data)
self.size = max(len(value) for value in data + [title])
self.title = self._pad(self.title)
for i, value in enumerate(self.data):
self.data[i] = self._pad(value)
def __str__(self):
return self.title
def __len__(self):
return len(self.data)
def __getitem__(self, key):
return self.data[key] if key < len(self.data) else [""]
def _transform(self, data):
return [
value if isinstance(value, str) else ", ".join(value)
for value in data
]
def _pad(self, s):
return s + " " * (self.size - len(s))
class RstTable():
def __init__(self, columns):
self.columns = columns
self.rowcount = max(len(col) for col in columns)
self.sep = "+" + "+".join("-" * col.size for col in columns) + "+"
def __iter__(self):
yield self.sep
yield "|" + "|".join(col.title for col in self.columns) + "|"
yield self.sep.replace("-", "=")
for i in range(self.rowcount):
yield self._format_row(i)
yield self.sep
def _format_row(self, row):
return "|" + "|".join(col[row] for col in self.columns) + "|"
def build_list():
exts = []
sub = []
last = None
for ex in gallery_dl.extractor.extractors():
c, sc = ex.category, [ex.subcategory, ex]
if c == last or not last:
sub.append(sc)
elif last:
if sub[0][0] and not (len(sub) == 1 and sub[0][0] == "image"):
sub.sort(key=sub_keys)
exts.append([last, sub])
sub = [sc]
last = c
exts.append([last, sorted(sub)])
for ext in exts:
ext[0] = map_category(ext[0])
for sub in ext[1]:
sub[0] = map_subcategory(sub[0])
exts.sort(key=lambda x: x[0].lower())
return exts
def get_domain(classes):
try: try:
categories[extr.category]["sc"].append(extr.subcategory) url = sys.modules[classes[0].__module__].__doc__.split()[-1]
except KeyError: if url.startswith("http"):
url = extr.__doc__.split()[-1] return url
if "." not in url[-5:-2]: except (IndexError, AttributeError):
url = sys.modules[extr.__module__].__doc__.split()[-1] pass
url = urllib.parse.urlsplit(url).netloc return ""
if url.startswith("www."):
url = url[4:]
categories[extr.category] = { def map_category(c):
"url": url, return CATEGORY_MAP.get(c, c.capitalize())
"sc": [extr.subcategory],
}
def map_subcategory(sc):
return SUBCATEGORY_MAP.get(sc, sc.capitalize() + "s")
def sub_keys(sc):
if sc[0] == "user":
return "A"
return sc[0]
exts = build_list()
columns = [
RstColumn("Site", [
ext[0]
for ext in exts
]),
RstColumn("URL", [
get_domain([subc[1] for subc in ext[1]])
for ext in exts
]),
RstColumn("Capabilities", [
", ".join(subc[0] for subc in ext[1])
for ext in exts
]),
]
outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst" outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst"
with open(os.path.join(ROOTDIR, outfile), "w") as file: with open(os.path.join(ROOTDIR, outfile), "w") as file:
file.write("Supported Sites\n" file.write("Supported Sites\n"
"===============\n") "===============\n")
for info in sorted(categories.values(), key=lambda x: x["url"]): for line in RstTable(columns):
file.write("- " + info["url"] + "\n") file.write(line + "\n")

204
supportedsites.rst
View File

@ -1,83 +1,125 @@
Supported Sites Supported Sites
=============== ===============
- 4chan.org +-------------------+---------------------------------------+----------------------------------------------------------+
- 8ch.net |Site |URL |Capabilities |
- bato.to +===================+=======================================+==========================================================+
- behoimi.org |3dbooru |http://behoimi.org/ |Pools, Posts, Tag-Searches |
- chan.sankakucomplex.com +-------------------+---------------------------------------+----------------------------------------------------------+
- chronos.to |4chan |https://www.4chan.org/ |Threads |
- coreimg.net +-------------------+---------------------------------------+----------------------------------------------------------+
- danbooru.donmai.us |8chan |https://8ch.net/ |Threads |
- deviantart.com +-------------------+---------------------------------------+----------------------------------------------------------+
- dynasty-scans.com |Batoto |https://bato.to/ |Chapters, Manga |
- e621.net +-------------------+---------------------------------------+----------------------------------------------------------+
- exhentai.org |Danbooru |https://danbooru.donmai.us/ |Pools, Posts, Tag-Searches |
- fapat.me +-------------------+---------------------------------------+----------------------------------------------------------+
- fascans.com |DeviantArt |https://www.deviantart.com/ |Images from Users, Favorites, individual Images |
- gelbooru.com +-------------------+---------------------------------------+----------------------------------------------------------+
- gomanga.co |Doki Reader |https://kobato.hologfx.com/ |Chapters, Manga |
- hbrowse.com +-------------------+---------------------------------------+----------------------------------------------------------+
- hentai2read.com |Dynasty Reader |https://dynasty-scans.com/ |Chapters |
- hentaifoundry.com +-------------------+---------------------------------------+----------------------------------------------------------+
- hentaihere.com |e621 |https://e621.net/ |Pools, Posts, Tag-Searches |
- hitomi.la +-------------------+---------------------------------------+----------------------------------------------------------+
- hosturimage.com |ExHentai |https://exhentai.org/ |Galleries |
- imagebam.com +-------------------+---------------------------------------+----------------------------------------------------------+
- imagefap.com |Fallen Angels Scans|https://fascans.com/ |Chapters |
- imageontime.org +-------------------+---------------------------------------+----------------------------------------------------------+
- imagetwist.com |Gelbooru |https://gelbooru.com/ |Posts, Tag-Searches |
- imagevenue.com +-------------------+---------------------------------------+----------------------------------------------------------+
- img.yt |GoManga |https://gomanga.co/ |Chapters, Manga |
- img4ever.net +-------------------+---------------------------------------+----------------------------------------------------------+
- imgbox.com |HBrowse |http://www.hbrowse.com/ |Chapters, Manga |
- imgcandy.net +-------------------+---------------------------------------+----------------------------------------------------------+
- imgchili.net |Hentai Foundry |https://www.hentai-foundry.com/ |Images from Users, individual Images |
- imgmaid.net +-------------------+---------------------------------------+----------------------------------------------------------+
- imgspice.com |Hentai2Read |https://hentai2read.com/ |Chapters, Manga |
- imgspot.org +-------------------+---------------------------------------+----------------------------------------------------------+
- imgth.com |HentaiHere |https://hentaihere.com/ |Chapters, Manga |
- imgtrex.com +-------------------+---------------------------------------+----------------------------------------------------------+
- imgtrial.com |Hitomi.la |https://hitomi.la/ |Galleries |
- imgupload.yt +-------------------+---------------------------------------+----------------------------------------------------------+
- imgur.com |ImageBam |http://www.imagebam.com/ |Galleries, individual Images |
- jaiminisbox.com +-------------------+---------------------------------------+----------------------------------------------------------+
- khinsider.com |ImageFap |http://imagefap.com/ |Images from Users, Galleries, individual Images |
- kisscomic.us +-------------------+---------------------------------------+----------------------------------------------------------+
- kissmanga.com |imgbox |https://imgbox.com/ |Galleries, individual Images |
- kobato.hologfx.com +-------------------+---------------------------------------+----------------------------------------------------------+
- konachan.com |imgChili |https://imgchili.net/ |Albums, individual Images |
- luscious.net +-------------------+---------------------------------------+----------------------------------------------------------+
- mangafox.me |imgth |https://imgth.com/ |Galleries |
- mangahere.co +-------------------+---------------------------------------+----------------------------------------------------------+
- mangapanda.com |imgur |https://imgur.com/ |Albums |
- mangapark.me +-------------------+---------------------------------------+----------------------------------------------------------+
- mangareader.net |Jaimini's Box |https://jaiminisbox.com/ |Chapters, Manga |
- mangashare.com +-------------------+---------------------------------------+----------------------------------------------------------+
- mangastream.com |Khinsider |https://downloads.khinsider.com/ |Soundtracks |
- nhentai.net +-------------------+---------------------------------------+----------------------------------------------------------+
- nijie.info |Kirei Cake |https://reader.kireicake.com/ |Chapters, Manga |
- pawoo.net +-------------------+---------------------------------------+----------------------------------------------------------+
- pic-maniac.com |KissComic |http://kisscomic.us/ |Comics, Comic-Issues |
- pinterest.com +-------------------+---------------------------------------+----------------------------------------------------------+
- pixhost.org |KissManga |http://kissmanga.com/ |Chapters, Manga |
- pixiv.net +-------------------+---------------------------------------+----------------------------------------------------------+
- postimg.org |Konachan |https://konachan.com/ |Pools, Posts, Tag-Searches |
- powermanga.org +-------------------+---------------------------------------+----------------------------------------------------------+
- rapidimg.net |Luscious |https://luscious.net/ |Albums |
- raw.senmanga.com +-------------------+---------------------------------------+----------------------------------------------------------+
- readcomiconline.to |Manga Fox |http://www.mangafox.me/ |Chapters |
- readcomics.tv +-------------------+---------------------------------------+----------------------------------------------------------+
- reader.kireicake.com |Manga Here |http://www.mangahere.co/ |Chapters, Manga |
- reader.seaotterscans.com +-------------------+---------------------------------------+----------------------------------------------------------+
- rule34.xxx |Manga Stream |https://mangastream.com/ |Chapters |
- safebooru.org +-------------------+---------------------------------------+----------------------------------------------------------+
- seiga.nicovideo.jp |Mangapanda |http://www.mangapanda.com/ |Chapters, Manga |
- sensescans.com +-------------------+---------------------------------------+----------------------------------------------------------+
- slide.world-three.org |MangaPark |http://mangapark.me/ |Chapters, Manga |
- thespectrum.net +-------------------+---------------------------------------+----------------------------------------------------------+
- tumblr.com |Mangareader |http://www.mangareader.net/ |Chapters, Manga |
- turboimagehost.com +-------------------+---------------------------------------+----------------------------------------------------------+
- twitter.com |Mangashare |http://www.mangashare.com/ |Chapters, Manga |
- yande.re +-------------------+---------------------------------------+----------------------------------------------------------+
- yomanga.co |nhentai |https://nhentai.net/ |Galleries |
- yonkouprod.com +-------------------+---------------------------------------+----------------------------------------------------------+
|Niconico Seiga |http://seiga.nicovideo.jp |Images from Users, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|nijie |https://nijie.info/ |Images from Users, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Pawoo |https://pawoo.net |Images from Users, Images from Statuses |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Pinterest |https://www.pinterest.com |Boards, Pins, Pinits |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Pixiv |https://www.pixiv.net/ |Images from Users, Bookmarks, Favorites, Individual Images|
+-------------------+---------------------------------------+----------------------------------------------------------+
|PowerManga |https://powermanga.org/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Read Comic Online |http://readcomiconline.to/ |Comics, Comic-Issues |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Readcomics |http://readcomics.tv/ |Comics, Comic-Issues |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Rule 34 |https://rule34.xxx/ |Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Safebooru |https://safebooru.org/ |Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sankaku Channel |https://chan.sankakucomplex.com/ |Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sea Otter Scans |https://reader.seaotterscans.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sen Manga |http://raw.senmanga.com/ |Chapters |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sense-Scans |http://sensescans.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Spectrum Nexus |http://www.thespectrum.net/manga_scans/|Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Tumblr |https://www.tumblr.com/ |Images from Users, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Twitter |https://twitter.com/ |Tweets |
+-------------------+---------------------------------------+----------------------------------------------------------+
|World Three |http://www.slide.world-three.org/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Yandere |https://yande.re/ |Pools, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|YoManga |https://yomanga.co/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Yonkou Productions |https://yonkouprod.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+