add simple progress indicator for multiple URLs (#19)
The output can be configured via the 'output.progress' config value. Possible values: - true: Show the default progress indicator "[{current}/{total}] {url}" (default) - false: Never show the progress indicator - <string>: Show the progress indicator using this as a custom format string(1). Possible replacement keys are: - current: current URL index - total : total number of URLs - url : current URL (1) https://docs.python.org/3/library/string.html#formatstrings
This commit is contained in:
parent
3ee77a0902
commit
d5a70f2580
@ -59,5 +59,11 @@
|
||||
"date-max": 253402210800,
|
||||
"recursion": 0
|
||||
}
|
||||
},
|
||||
"output":
|
||||
{
|
||||
"mode": "auto",
|
||||
"shorten": true,
|
||||
"progress": true
|
||||
}
|
||||
}
|
||||
|
@ -41,6 +41,15 @@ def initialize_logging():
|
||||
root.addHandler(handler)
|
||||
|
||||
|
||||
def progress(urls, pformat):
|
||||
if pformat is True:
|
||||
pformat = "[{current}/{total}] {url}"
|
||||
pinfo = {"total": len(urls)}
|
||||
for pinfo["current"], pinfo["url"] in enumerate(urls, 1):
|
||||
print(pformat.format_map(pinfo), file=sys.stderr)
|
||||
yield pinfo["url"]
|
||||
|
||||
|
||||
def sanatize_input(file):
|
||||
for line in file:
|
||||
line = line.strip()
|
||||
@ -98,8 +107,7 @@ def main():
|
||||
file = sys.stdin
|
||||
else:
|
||||
file = open(args.inputfile)
|
||||
import itertools
|
||||
urls = itertools.chain(urls, sanatize_input(file))
|
||||
urls += sanatize_input(file)
|
||||
except OSError as exc:
|
||||
log.warning("input-file: %s", exc)
|
||||
|
||||
@ -109,6 +117,10 @@ def main():
|
||||
except OSError as exc:
|
||||
log.warning("unsupported-URL file: %s", exc)
|
||||
|
||||
pformat = config.get(("output", "progress"), True)
|
||||
if pformat and len(urls) > 1 and args.loglevel < logging.ERROR:
|
||||
urls = progress(urls, pformat)
|
||||
|
||||
for url in urls:
|
||||
try:
|
||||
log.debug("Starting %s for '%s'", jobtype.__name__, url)
|
||||
|
@ -57,7 +57,7 @@ class PawooAccountExtractor(PawooExtractor):
|
||||
test = [
|
||||
("https://pawoo.net/@kuroda", {
|
||||
"url": "a3f9e7555f2b024554c0e9b6cbcc7991af13cf99",
|
||||
"keyword": "81e084503755d564ef15ef0db88ae1d9cf89f258",
|
||||
"keyword": "1b4e0dc5ac6c22ce9485ba12ecc200d0aaa2ffae",
|
||||
}),
|
||||
("https://pawoo.net/@zZzZz/", {
|
||||
"exception": exception.NotFoundError,
|
||||
@ -85,7 +85,7 @@ class PawooStatusExtractor(PawooExtractor):
|
||||
test = [
|
||||
("https://pawoo.net/@takehana_note/559043", {
|
||||
"url": "f95cc8c0274c4143e7e21dbdc693b90c65b596e3",
|
||||
"keyword": "590105849b0435e17ca65258b9150cd0502d24da",
|
||||
"keyword": "7d060d9c4572b381aa423797ad48d89a12daac77",
|
||||
"content": "3b148cf90174173355fe34179741ce476921b2fc",
|
||||
}),
|
||||
("https://pawoo.net/@zZzZz/12346", {
|
||||
|
Loading…
x
Reference in New Issue
Block a user