From f2000a69aa4f2de16c397c8ffa801a26d2e0aa2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sat, 29 Jun 2019 22:48:59 +0200
Subject: [PATCH] implement 'image-unique' and 'chapter-unique' options (#303)

The default value for both is 'false', i.e. duplicate URLs are NOT
ignored.

The previous behavior was to always ignore duplicate URLs to make
'--abort-on-skip' work properly when new images where added to the
beginning of a collection while gallery-dl is running.
---
 docs/configuration.rst | 22 ++++++++++++++++++++++
 gallery_dl/job.py      | 17 ++++++++---------
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index 4db8281f..db8fa0f7 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -381,6 +381,28 @@ Description Like `image-filter`__, but applies to delegated URLs
 __ `extractor.*.image-filter`_
 
 
+extractor.*.image-unique
+------------------------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Ignore image URLs that have been encountered before during the
+            current extractor run.
+=========== =====
+
+
+extractor.*.chapter-unique
+--------------------------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Like `image-unique`__, but applies to delegated URLs
+            like manga-chapters, etc.
+=========== =====
+
+__ `extractor.*.image-unique`_
+
+
 
 Extractor-specific Options
 ==========================
diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 749bde72..978b66ca 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -29,15 +29,9 @@ class Job():
         extr.log.job = self
         extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url)
 
-        # url predicates
-        self.pred_url = self._prepare_predicates(
-            "image", [util.UniquePredicate()], True)
+        self.pred_url = self._prepare_predicates("image", True)
+        self.pred_queue = self._prepare_predicates("chapter", False)
 
-        # queue predicates
-        self.pred_queue = self._prepare_predicates(
-            "chapter", [], False)
-
-        # category transfer
         if parent and parent.extractor.config(
                 "category-transfer", parent.extractor.categorytransfer):
             self.extractor.category = parent.extractor.category
@@ -142,7 +136,12 @@ class Job():
         if self.userkwds:
             kwdict.update(self.userkwds)
 
-    def _prepare_predicates(self, target, predicates, skip=True):
+    def _prepare_predicates(self, target, skip=True):
+        predicates = []
+
+        if self.extractor.config(target + "-unique"):
+            predicates.append(util.UniquePredicate())
+
         pfilter = self.extractor.config(target + "-filter")
         if pfilter:
             try: