From 3ecf4a97299497ab92862185a70e62c2504f4783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Fri, 15 Oct 2021 19:00:04 +0200
Subject: [PATCH] update gallery-dl-example.conf

- add comments
- remove/replace some older options
- display more post processor settings
---
 docs/gallery-dl-example.conf | 440 +++++++++++++++++++++++------------
 1 file changed, 286 insertions(+), 154 deletions(-)

diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf
index d84febde..7d2eea36 100644
--- a/docs/gallery-dl-example.conf
+++ b/docs/gallery-dl-example.conf
@@ -2,156 +2,11 @@
     "extractor":
     {
         "base-directory": "~/gallery-dl/",
+
+        "#": "set global archive file for all extractors",
         "archive": "~/gallery-dl/archive.sqlite3",
-        "proxy": "http://10.10.1.10:3128",
 
-        "postprocessors": [
-            {
-                "name": "ugoira",
-                "whitelist": ["pixiv", "danbooru"],
-                "ffmpeg-twopass": true,
-                "ffmpeg-args": ["-c:v", "libvpx", "-crf", "4", "-b:v", "5000k", "-an"]
-            },
-            {
-                "name": "metadata",
-                "whitelist": ["danbooru", "yandere", "sankaku"],
-                "mode": "tags"
-            }
-        ],
-
-        "pixiv":
-        {
-            "archive": "~/gallery-dl/archive-pixiv.sqlite3",
-
-            "filename": "{id}{num}.{extension}",
-            "directory": ["Pixiv", "Works", "{user[id]}"],
-
-            "username": "foo",
-            "password": "bar",
-
-            "favorite":
-            {
-                "directory": ["Pixiv", "Favorites", "{user[id]}"]
-            },
-
-            "bookmark":
-            {
-                "directory": ["Pixiv", "My Bookmarks"],
-
-                "username": "foo123",
-                "password": "bar123"
-            }
-        },
-
-        "exhentai":
-        {
-            "cookies":
-            {
-                "ipb_member_id": "12345",
-                "ipb_pass_hash": "1234567890abcdef",
-                "igneous"      : "123456789",
-                "hath_perks"   : "m1.m2.m3.a-123456789a"
-            },
-
-            "proxy":
-            {
-                "http": "http://10.10.1.10:8080",
-                "https": "https://10.10.1.10:443"
-            },
-
-            "filename": "{num:>04}_{name}.{extension}",
-            "directory": ["{category!c}", "{title}"],
-
-            "wait-min": 1.0,
-            "wait-max": 5.0
-        },
-
-        "mangadex":
-        {
-            "chapter-filter": "lang == 'en'",
-            "postprocessors": [{
-                "name": "zip",
-                "keep-files": false,
-                "compression": "zip"
-            }]
-        },
-
-        "deviantart":
-        {
-            "include": "gallery,scraps",
-            "metadata": true,
-            "postprocessors": [{
-                "name": "metadata",
-                "mode": "custom",
-                "directory"       : "Descriptions",
-                "content-format"  : "{description}\n",
-                "extension-format": "descr.txt"
-            }]
-        },
-
-        "flickr":
-        {
-            "access-token": "1234567890-abcdef",
-            "access-token-secret": "1234567890abcdef",
-            "size-max": 1920
-        },
-
-        "reddit":
-        {
-            "morecomments": true,
-            "date-min": "2017-01",
-            "date-format": "%Y-%m",
-            "recursion": 1
-        },
-
-        "sankaku":
-        {
-            "sleep": 2,
-            "wait-min": 5.0,
-            "wait-max": 5.0,
-            "cookies": "~/gallery-dl/cookies-sankaku.txt"
-        },
-
-        "tumblr":
-        {
-            "posts": "all",
-            "external": false,
-            "reblogs": false,
-            "inline": true,
-
-            "likes":
-            {
-                "posts": "video,photo,link",
-                "external": true,
-                "reblogs": true
-            }
-        },
-
-        "mastodon":
-        {
-            "mastodon.xyz":
-            {
-                "access-token": "cab65529..."
-            },
-            "tabletop.social": {
-                "access-token": "513a36c6..."
-            },
-
-            "directory": ["mastodon", "{instance}", "{account[username]!l}"],
-            "filename": "{id}_{media[id]}.{extension}"
-        },
-
-        "foolslide": {
-            "otscans":   {"root": "https://otscans.com/foolslide"},
-            "helvetica": {"root": "https://helveticascans.com/r" }
-        },
-
-        "foolfuuka": {
-            "fireden-onion": {"root": "http://ydt6jy2ng3s3xg2e.onion"},
-            "scalearchive":  {"root": "https://archive.scaled.team"  }
-        },
-
-        "replace invalid path characters with unicode alternatives": null,
+        "#": "replace invalid path characters with unicode alternatives",
         "path-restrict": {
             "\\": "⧹",
             "/" : "⧸",
@@ -162,22 +17,237 @@
             "\"": "″",
             "<" : "﹤",
             ">" : "﹥"
+        },
+
+        "#": "write tags for several *booru sites",
+        "postprocessors": [
+            {
+                "name": "metadata",
+                "mode": "tags",
+                "whitelist": ["danbooru", "moebooru", "sankaku"]
+            }
+        ],
+
+        "pixiv":
+        {
+            "#": "override global archive setting for pixiv",
+            "archive": "~/gallery-dl/archive-pixiv.sqlite3",
+
+            "#": "set custom directory and filename format strings for all pixiv downloads",
+            "filename": "{id}{num}.{extension}",
+            "directory": ["Pixiv", "Works", "{user[id]}"],
+            "refresh-token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ01234567890-FedC9",
+
+            "#": "transform ugoira into lossless MKVs",
+            "ugoira": true,
+            "postprocessors": ["ugoira-copy"],
+
+            "#": "use special settings for favorites and bookmarks",
+            "favorite":
+            {
+                "directory": ["Pixiv", "Favorites", "{user[id]}"]
+            },
+            "bookmark":
+            {
+                "directory": ["Pixiv", "My Bookmarks"],
+                "refresh-token": "01234567890aBcDeFgHiJkLmNoPqRsTuVwXyZ-ZyxW1"
+            }
+        },
+
+        "danbooru":
+        {
+            "ugoira": true,
+            "postprocessors": ["ugoira-webm"]
+        },
+
+        "exhentai":
+        {
+            "#": "use cookies instead of logging in with username and password",
+            "cookies":
+            {
+                "ipb_member_id": "12345",
+                "ipb_pass_hash": "1234567890abcdef",
+                "igneous"      : "123456789",
+                "hath_perks"   : "m1.m2.m3.a-123456789a",
+                "sk"           : "n4m34tv3574m2c4e22c35zgeehiw",
+                "sl"           : "dm_2"
+            },
+
+            "#": "wait 2 to 4.8 seconds between HTTP requests",
+            "sleep-request": [2.0, 4.8],
+
+            "filename": "{num:>04}_{name}.{extension}",
+            "directory": ["{category!c}", "{title}"]
+        },
+
+        "sankaku":
+        {
+            "#": "authentication with cookies is not possible for sankaku",
+            "username": "user",
+            "password": "#secret#"
+        },
+
+        "furaffinity": {
+            "#": "authentication with username and password is not possible due to CAPTCHA",
+            "cookies": {
+                "a": "01234567-89ab-cdef-fedc-ba9876543210",
+                "b": "fedcba98-7654-3210-0123-456789abcdef"
+            },
+
+            "descriptions": "html",
+            "postprocessors": ["content"]
+        },
+
+        "deviantart":
+        {
+            "#": "download 'gallery' and 'scraps' images for user profile URLs",
+            "include": "gallery,scraps",
+
+            "#": "use custom API credentials to avoid 429 errors",
+            "client-id": "98765",
+            "client-secret": "0123456789abcdef0123456789abcdef",
+            "refresh-token": "0123456789abcdef0123456789abcdef01234567",
+
+            "#": "put description texts into a separate directory",
+            "metadata": true,
+            "postprocessors": [
+                {
+                    "name": "metadata",
+                    "mode": "custom",
+                    "directory"       : "Descriptions",
+                    "content-format"  : "{description}\n",
+                    "extension-format": "descr.txt"
+                }
+            ]
+        },
+
+        "flickr":
+        {
+            "access-token": "1234567890-abcdef",
+            "access-token-secret": "1234567890abcdef",
+            "size-max": 1920
+        },
+
+        "mangadex":
+        {
+            "#": "only download safe/suggestive chapters translated to English",
+            "lang": "en",
+            "ratings": ["safe", "suggestive"],
+
+            "#": "put chapters into '.cbz' archives",
+            "postprocessors": ["cbz"]
+        },
+
+        "reddit":
+        {
+            "#": "put files from child extractors into the reddit directory",
+            "parent-directory": true,
+
+            "#": "transfer metadata to any child extractor as '_reddit'",
+            "parent-metadata": "_reddit",
+
+            "#": "only spawn child extractors for links to specific sites",
+            "whitelist": ["imgur", "redgifs", "gfycat"]
+        },
+
+        "imgur":
+        {
+            "#": "use different directory and filename formats when coming from a reddit post",
+            "directory":
+            {
+                "locals().get('_reddit')": []
+            },
+            "filename":
+            {
+                "locals().get('_reddit')": "{_reddit[id]} {id}.{extension}",
+                ""                     : "{id}.{extension}"
+            }
+        },
+
+        "tumblr":
+        {
+            "posts": "all",
+            "external": false,
+            "reblogs": false,
+            "inline": true,
+
+            "#": "use special settings when downloading liked posts",
+            "likes":
+            {
+                "posts": "video,photo,link",
+                "external": true,
+                "reblogs": true
+            }
+        },
+
+        "twitter":
+        {
+            "#": "write text content for *all* tweets",
+            "postprocessors": ["content"],
+            "text-tweets": true
+        },
+
+        "mastodon":
+        {
+            "#": "add 'tabletop.social' as recognized mastodon instance",
+            "#": "(run 'gallery-dl oauth:mastodon:tabletop.social to get an access token')",
+            "tabletop.social":
+            {
+                "root": "https://tabletop.social",
+                "access-token": "513a36c6..."
+            },
+
+            "#": "set filename format strings for all 'mastodon' instances",
+            "directory": ["mastodon", "{instance}", "{account[username]!l}"],
+            "filename" : "{id}_{media[id]}.{extension}"
+        },
+
+        "foolslide": {
+            "#": "add two more foolslide instances",
+            "otscans"  : {"root": "https://otscans.com/foolslide"},
+            "helvetica": {"root": "https://helveticascans.com/r" }
+        },
+
+        "foolfuuka": {
+            "#": "add two other foolfuuka 4chan archives",
+            "fireden-onion": {"root": "http://ydt6jy2ng3s3xg2e.onion"},
+            "scalearchive" : {"root": "https://archive.scaled.team"  }
         }
     },
 
     "downloader":
     {
-        "part-directory": "/tmp/.download/",
+        "#": "restrict download speed to 1 MB/s",
         "rate": "1M",
+
+        "#": "show download progress indicator after 2 seconds",
+        "progress": 2.0,
+
+        "#": "retry failed downloads up to 3 times",
         "retries": 3,
-        "timeout": 8.5
+
+        "#": "consider a download 'failed' after 8 seconds of inactivity",
+        "timeout": 8.0,
+
+        "#": "write '.part' files into a special directory",
+        "part-directory": "/tmp/.download/",
+
+        "#": "do not update file modification times",
+        "mtime": false,
+
+        "ytdl":
+        {
+            "#": "use yt-dlp instead of youtube-dl",
+            "module": "yt_dlp"
+        }
     },
 
     "output":
     {
-        "mode": "terminal",
         "log": {
             "level": "info",
+
+            "#": "use different ANSI colors for each log level",
             "format": {
                 "debug"  : "\u001b[0;37m{name}: {message}\u001b[0m",
                 "info"   : "\u001b[1;37m{name}: {message}\u001b[0m",
@@ -185,11 +255,19 @@
                 "error"  : "\u001b[1;31m{name}: {message}\u001b[0m"
             }
         },
+
+        "#": "shorten filenames to fit into one terminal line",
+        "#": "while also considering wider East-Asian characters",
+        "shorten": "eaw",
+
+        "#": "write logging messages to a separate file",
         "logfile": {
             "path": "~/gallery-dl/log.txt",
             "mode": "w",
             "level": "debug"
         },
+
+        "#": "write unrecognized URLs to a separate file",
         "unsupportedfile": {
             "path": "~/gallery-dl/unsupported.txt",
             "mode": "a",
@@ -198,9 +276,63 @@
         }
     },
 
-    "cache": {
-        "file": "~/gallery-dl/cache.sqlite3"
+    "postprocessor":
+    {
+        "#": "write 'content' metadata into separate files",
+        "content":
+        {
+            "name" : "metadata",
+
+            "#": "write data for every post instead of each individual file",
+            "event": "post",
+            "filename": "{post_id|tweet_id|id}.txt",
+
+            "#": "write only the values for 'content' or 'description'",
+            "mode" : "custom",
+            "format": "{content|description}\n"
+        },
+
+        "#": "put files into a '.cbz' archive",
+        "cbz":
+        {
+            "name": "zip",
+            "extension": "cbz"
+        },
+
+        "#": "various ugoira post processor configurations to create different file formats",
+        "ugoira-webm":
+        {
+            "name": "ugoira",
+            "extension": "webm",
+            "ffmpeg-args": ["-c:v", "libvpx-vp9", "-an", "-b:v", "0", "-crf", "30"],
+            "ffmpeg-twopass": true,
+            "ffmpeg-demuxer": "image2"
+        },
+        "ugoira-mp4":
+        {
+            "name": "ugoira",
+            "extension": "mp4",
+            "ffmpeg-args": ["-c:v", "libx264", "-an", "-b:v", "4M", "-preset", "veryslow"],
+            "ffmpeg-twopass": true,
+            "libx264-prevent-odd": true
+        },
+        "ugoira-gif":
+        {
+            "name": "ugoira",
+            "extension": "gif",
+            "ffmpeg-args": ["-filter_complex", "[0:v] split [a][b];[a] palettegen [p];[b][p] paletteuse"]
+        },
+        "ugoira-copy": {
+            "name": "ugoira",
+            "extension": "mkv",
+            "ffmpeg-args": ["-c", "copy"],
+            "libx264-prevent-odd": false,
+            "repeat-last-frame": false
+        }
     },
 
-    "netrc": true
+    "#": "use a custom cache file location",
+    "cache": {
+        "file": "~/gallery-dl/cache.sqlite3"
+    }
 }