#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2019 A S Lewis
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see .
"""Download operation classes."""
# Import Gtk modules
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import GObject
# Import other modules
import datetime
import json
import signal
import os
import queue
import requests
import subprocess
import sys
import threading
import time
# Import our modules
from . import constants
from . import mainapp
from . import media
from . import options
from . import utils
# Decorator to add thread synchronisation to some functions in the
# downloads.DownloadList object
_SYNC_LOCK = threading.RLock()
def synchronise(lock):
def _decorator(func):
def _wrapper(*args, **kwargs):
lock.acquire()
ret_value = func(*args, **kwargs)
lock.release()
return ret_value
return _wrapper
return _decorator
# Classes
class DownloadManager(threading.Thread):
"""Called by mainapp.TartubeApp.download_manager_start().
Based on the DownloadManager class in youtube-dl-gui.
Python class to manage a download operation.
Creates one or more downloads.DownloadWorker objects, each of which handles
a single download.
This object runs on a loop, looking for available workers and, when one is
found, assigning them something to download. The worker completes that
download and then waits for another assignment.
Args:
app_obj: The mainapp.TartubeApp object
force_sim_flag (True/False): True if playlists/channels should just be
checked for new videos, without downloading anything. False if
videos should be downloaded (or not) depending on each media data
object's .dl_sim_flag IV
download_list_obj(downloads.DownloadManager): An ordered list of
media data objects to download, each one represented by a
downloads.DownloadItem object
"""
# Standard class methods
def __init__(self, app_obj, force_sim_flag, download_list_obj):
super(DownloadManager, self).__init__()
# IV list - class objects
# -----------------------
# The mainapp.TartubeApp object
self.app_obj = app_obj
# Each instance of this object, which represents a single download
# operation, creates its own options.OptionsParser object. That
# object convert the download options stored in
# downloads.DownloadWorker.options_list into a list of youtube-dl
# command line options
self.options_parser_obj = None
# An ordered list of media data objects to download, each one
# represented by a downloads.DownloadItem object
self.download_list_obj = download_list_obj
# List of downloads.DownloadWorker objects, each one handling one of
# several simultaneous downloads
self.worker_list = []
# IV list - other
# ---------------
# Flag set to True if playlists/channels should just be checked for new
# videos, without downloading anything. False if videos should be
# downloaded (or not) depending on each media data object's
# .dl_sim_flag IV
self.force_sim_flag = force_sim_flag
# The time at which the download operation began (in seconds since
# epoch)
self.start_time = time.time()
# The time at which the download operation completed (in seconds since
# epoch)
self.stop_time = None
# The time (in seconds) between iterations of the loop in self.run
self.sleep_time = 0.1
# Flag set to False if self.stop_download_operation() is called
# The False value halts the main loop in self.run()
self.running_flag = True
# Number of download jobs started (number of downloads.DownloadItem
# objects which have been allocated to a worker)
self.job_count = 0
# Code
# ----
# Create an object for converting download options stored in
# downloads.DownloadWorker.options_list into a list of youtube-dl
# command line options
self.options_parser_obj = options.OptionsParser(self)
# Create a list of downloads.DownloadWorker objects, each one handling
# one of several simultaneous downloads
for i in range(1, self.app_obj.num_worker_default + 1):
self.worker_list.append(DownloadWorker(self))
# Let's get this party started!
self.start()
# Public class methods
def run(self):
"""Called as a result of self.__init__().
On a continuous loop, passes downloads.DownloadItem objects to each
downloads.DownloadWorker object, as they become available, until the
download operation is complete.
"""
# Perform the download operation until there is nothing left to
# download, or until something has called
# self.stop_download_operation()
while self.running_flag:
download_item_obj = self.download_list_obj.fetch_next_item()
# Exit this loop when there are no more downloads.DownloadItem
# objects whose .status is constants.MAIN_STAGE_QUEUED, and when
# all workers have finished their downloads
# Otherwise, wait for an available downloads.DownloadWorker, and
# then assign the next downloads.DownloadItem to it
if not download_item_obj:
if self.check_workers_all_finished():
break
else:
worker_obj = self.get_available_worker()
if worker_obj:
# If the worker has been marked as doomed (because the
# number of simultaneous downloads allowed has decreased)
# then we can destroy it now
if worker_obj.doomed_flag:
worker_obj.close()
self.remove_worker(worker_obj)
# Otherwise, initialise the worker's IVs for the next job
else:
worker_obj.prepare_download(download_item_obj)
# Change the download stage for that
# downloads.DownloadItem
self.download_list_obj.change_item_stage(
download_item_obj.dbid,
constants.MAIN_STAGE_ACTIVE,
)
# Update the main window's progress bar
self.job_count += 1
self.app_obj.main_win_obj.update_progress_bar(
download_item_obj.media_data_obj.name,
self.job_count,
len(self.download_list_obj.download_item_list),
)
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
# Download operation complete (or has been stopped)
# Close all the workers
for worker_obj in self.worker_list:
worker_obj.close()
# Join and collect
for worker_obj in self.worker_list:
worker_obj.join()
# Set the stop time
self.stop_time = time.time()
# Tell the Progress Tab to display any remaining download statistics
# immediately
self.app_obj.main_win_obj.progress_list_display_dl_stats()
# When youtube-dl reports it is finished, there is a short delay before
# the final downloaded video(s) actually exist in the filesystem
# Therefore, mainwin.MainWin.progress_list_display_dl_stats() may not
# have marked the final video(s) as downloaded yet
# Let the timer run for a few more seconds to allow those videos to be
# marked as downloaded (we can stop before that, if all the videos
# have been already marked)
if not self.force_sim_flag:
self.app_obj.download_manager_halt_timer()
else:
# If we're only simulating downloads, we don't need to wait at all
self.app_obj.download_manager_finished()
def change_worker_count(self, number):
"""Called by mainapp.TartubeApp.set_num_worker_default().
When the number of simultaneous downloads allowed is changed during a
download operation, this function responds.
If the number has increased, creates an extra download worker object.
If the number has decreased, marks the worker as doomed. When its
current download is completed, the download manager destroys it.
Args:
number (int): The new value of
mainapp.TartubeApp.num_worker_default
"""
# How many workers do we have already?
current = len(self.worker_list)
# If this object hasn't set up its worker pool yet, let the setup code
# proceed as normal
# Sanity check: if the specified value is less than 1, or hasn't
# changed, take no action
if not current or number < 1 or current == number:
return
# Usually, the number of workers goes up or down by one at a time, but
# we'll check for larger leaps anyway
for i in range(1, (abs(current-number) + 1)):
if number > current:
# The number has increased. If any workers have marked as
# doomed, they can be unmarked, allowing them to continue
match_flag = False
for worker_obj in self.worker_list:
if worker_obj.doomed_flag:
worker_obj.set_doomed_flag(True)
match_flag = True
break
if not match_flag:
# No workers were marked doomed, so create a brand new
# download worker
self.worker_list.append(DownloadWorker(self))
else:
# The number has decreased. The first worker in the list is
# marked as doomed - that is, when it has finished its
# current job, it closes (rather than being given another
# job, as usual)
for worker_obj in self.worker_list:
if not worker_obj.doomed_flag:
worker_obj.set_doomed_flag(True)
break
def check_workers_all_finished(self):
"""Called by self.run().
Based on DownloadManager._jobs_done().
Returns:
True if all downloads.DownloadWorker objects have finished their
jobs, otherwise returns False
"""
for worker_obj in self.worker_list:
if not worker_obj.available_flag:
return False
return True
def get_available_worker(self):
"""Called by self.run().
Based on DownloadManager._get_worker().
Returns:
The first available downloads.DownloadWorker, or None if there are
no available workers.
"""
for worker_obj in self.worker_list:
if worker_obj.available_flag:
return worker_obj
return None
def remove_worker(self, worker_obj):
"""Called by self.run().
When a worker marked as doomed has completed its download job, this
function is called to remove it from self.worker_list.
Args:
worker_obj (downloads.DownloadWorker): The worker object to remove
"""
new_list = []
for other_obj in self.worker_list:
if other_obj != worker_obj:
new_list.append(other_obj)
self.worker_list = new_list
def stop_download_operation(self):
"""Called by mainapp.TartubeApp.do_shutdown(), .stop() and a callback
in .on_button_stop_operation().
Based on DownloadManager.stop_downloads().
Stops the download operation. On the next iteration of self.run()'s
loop, the downloads.DownloadWorker objects are cleaned up.
"""
self.running_flag = False
class DownloadWorker(threading.Thread):
"""Called by downloads.DownloadManager.__init__().
Based on the Worker class in youtube-dl-gui.
Python class for managing simultaneous downloads. The parent
downloads.DownloadManager object can create one or more workers, each of
which handles a single download.
The download manager runs on a loop, looking for available workers and,
when one is found, assigns them something to download. The worker
completes that download and then waits for another assignment.
Args:
download_manager_obj(downloads.DownloadManager): The parent download
manager object.
"""
# Standard class methods
def __init__(self, download_manager_obj):
super(DownloadWorker, self).__init__()
# IV list - class objects
# -----------------------
# The parent downloads.DownloadManager object
self.download_manager_obj = download_manager_obj
# The downloads.DownloadItem object for the current job
self.download_item_obj = None
# The downloads.VideoDownloader object for the current job
self.video_downloader_obj = None
# The options.OptionsManager object for the current job
self.options_manager_obj = None
# IV list - other
# ---------------
# The time (in seconds) between iterations of the loop in self.run
self.sleep_time = 0.1
# Flag set to False if self.close() is called
# The False value halts the main loop in self.run()
self.running_flag = True
# Flag set to True when the parent downloads.DownloadManager object
# wants to destroy this worker, having called self.set_doomed_flag()
# to do that
# The worker is not destroyed until its current download is complete
self.doomed_flag = False
# Options list (used by downloads.VideoDownloader)
# Initialised in the call to self.prepare_download()
self.options_list = []
# Flag set to True when the worker is available for a new job, False
# when it is already occupied with a job
self.available_flag = True
# Code
# ----
# Let's get this party started!
self.start()
# Public class methods
def run(self):
"""Called as a result of self.__init__().
Waits until this worker has been assigned a job, at which time we
create a new downloads.VideoDownloader object and wait for the result.
"""
# Import the main application (for convenience)
app_obj = self.download_manager_obj.app_obj
# Handle a job, or wait for the downloads.DownloadManager to assign
# this worker a job
while self.running_flag:
# If this worker is currently assigned a job...
if not self.available_flag:
# youtube-dl-gui used a single instance of a
# YoutubeDLDownloader object for each instance of a Worker
# object.
# This causes problems, so Tartube will use a new
# downloads.VideoDownloader object each time
# Set up the new downloads.VideoDownloader object
self.video_downloader_obj = VideoDownloader(
self.download_manager_obj,
self,
self.download_item_obj,
)
# Then execute the assigned job
return_code = self.video_downloader_obj.do_download()
# If the downloads.VideoDownloader object collected any
# youtube-dl error/warning messages, display them in the
# Error List
if self.download_item_obj.media_data_obj.error_list \
or self.download_item_obj.media_data_obj.warning_list:
app_obj.main_win_obj.errors_list_add_row(
self.download_item_obj.media_data_obj,
)
# In the event of an error, nothing updates the video's row in
# the Video Catalogue, and therefore the error icon won't be
# visible
# Do that now (but don't both if mainwin.ComplexCatalogueItem
# aren't being used in the Video Catalogue)
if return_code == VideoDownloader.ERROR \
and app_obj.complex_catalogue_flag:
app_obj.main_win_obj.video_catalogue_update_row(
self.download_item_obj.media_data_obj,
)
# Call the destructor function of VideoDownloader object
self.video_downloader_obj.close()
# This worker is now available for a new job
self.available_flag = True
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
def close(self):
"""Called by downloads.DownloadManager.run().
This worker object is closed when:
1. The download operation is complete (or has been stopped)
2. The worker has been marked as doomed, and the calling function
is now ready to destroy it
Tidy up IVs and stop any child processes.
"""
self.running_flag = False
if self.video_downloader_obj:
self.video_downloader_obj.stop()
def prepare_download(self, download_item_obj):
"""Called by downloads.DownloadManager.run().
Based on Worker.download().
Updates IVs for a new job, so that self.run can initiate the download.
Args:
download_item_obj (downloads.DownloadItem): The download item
object describing the URL from which youtube-dl should download
video(s).
"""
self.download_item_obj = download_item_obj
self.options_manager_obj = download_item_obj.options_manager_obj
self.options_list = self.download_manager_obj.options_parser_obj.parse(
download_item_obj,
self.options_manager_obj.options_dict,
)
self.available_flag = False
def set_doomed_flag(self, flag):
"""Called by downloads.DownloadManager.change_worker_count()."""
self.doomed_flag = flag
# Callback class methods
def data_callback(self, dl_stat_dict):
"""Called by downloads.VideoDownloader.do_download() and
.last_data_callback().
Based on Worker._data_hook() and ._talk_to_gui().
'dl_stat_dict' holds a dictionary of statistics in a standard format
specified by downloads.VideoDownloader.extract_stdout_data().
This callback receives that dictionary and passes it on to the main
window, so the statistics can be displayed there.
Args:
dl_stat_dict (dictionary): The dictionary of statistics described
above
"""
app_obj = self.download_manager_obj.app_obj
app_obj.main_win_obj.progress_list_receive_dl_stats(
self.download_item_obj,
dl_stat_dict,
)
class DownloadList(object):
"""Called by mainapp.TartubeApp.download_manager_start().
Based on the DownloadList class in youtube-dl-gui.
Python class to keep track of all the media data objects to be downloaded
(for real or in simulation) during a downloaded operation.
This object contains an ordered list of downloads.DownloadItem objects.
Each of those objects represents a media data object to be downloaded
(media.Video, media.Channel, media.Playlist or media.Folder).
Videos are downloaded in the order specified by the list.
Args:
app_obj (mainapp.AxtubeApp): The main application
media_data_obj (media.Video, media.Channel, media.Playlist,
media.Folder or None): The media data object to download. If
specified, that object and any media data objects it contains are
downloaded. If none, all media data objects in Tartube's media data
registry are downloaded
"""
# Standard class methods
def __init__(self, app_obj, media_data_obj):
# IV list - class objects
# -----------------------
self.app_obj = app_obj
# IV list - other
# ---------------
# Number of download.DownloadItem objects created (used to give each a
# unique ID)
self.download_item_count = 0
# An ordered list of downloads.DownloadList items, one for each
# media.Video, media.Channel, media.Playlist or media.Folder object
# This list stores each item's .dbid
self.download_item_list = []
# Corresponding dictionary of downloads.DownloadList items for quick
# lookup. Dictionary in the form
# key = download.DownloadItem.dbid
# value = the download.DownloadItem object itself
self.download_item_dict = {}
# Code
# ----
# For each media data object to be downloaded, created a
# downloads.DownloadItem object, and update the IVs above
if not media_data_obj:
# Use all media data objects
for dbid in self.app_obj.media_top_level_list:
obj = self.app_obj.media_reg_dict[dbid]
self.create_item(obj)
elif isinstance(media_data_obj, media.Folder) \
and media_data_obj.priv_flag:
# Videos in a private folder's .child_list can't be downloaded
# (since they are also a child of a channel, playlist or a public
# folder)
self.app_obj.system_error(
301,
'Cannot download videos in a private folder',
)
else:
# Use the specified media data object. The True value tells
# self.create_item to download media_data_obj, even if it is a
# video in a channel or a playlist (which otherwise would be
# handled by downloading the channel/playlist)
self.create_item(media_data_obj, True)
# Public class methods
@synchronise(_SYNC_LOCK)
def change_item_stage(self, dbid, new_stage):
"""Called by downloads.DownloadManager.run().
Based on DownloadList.change_stage().
Changes the download stage for the specified downloads.DownloadItem
object.
Args:
dbid (int): The specified item's .dbid
new_stage: The new download stage, one of the values imported from
constants.py (e.g. constants.MAIN_STAGE_QUEUED)
"""
self.download_item_dict[dbid].stage = new_stage
def create_item(self, media_data_obj, init_flag=False):
"""Called by self.__init__(), or by this function recursively.
Creates a downloads.DownloadItem object for media data objects in the
media data registry.
Doesn't create a download item object for:
- media.Video objects whose parent is not a media.Folder (i.e.
whose parent is a media.Channel or a media.Playlist)
- media.Video objects in any restricted folder
- media.Video objects in the fixed 'Unsorted Videos' folder which
are already marked as downloaded
- media.Folder objects
Adds the resulting downloads.DownloadItem object to this object's IVs.
Args:
media_data_obj (media.Video, media.Channel, media.Playlist,
media.Folder): A media data object
init_flag (True, False): True when called by self.__init__, and
False when called by this function recursively. If True and
media_data_obj is a media.Video object, we download it even if
its parent is a channel or a playlist
"""
# Get the options.OptionsManager object that applies to this media
# data object
# (The manager might be specified by obj itself, or it might be
# specified by obj's parent, or we might use the default
# options.OptionsManager)
options_manager_obj = self.get_options_manager(media_data_obj)
# Ignore private folders, and don't download any of their children
# (because they are all children of some other non-private folder)
if isinstance(media_data_obj, media.Folder) \
and media_data_obj.priv_flag:
return
# Don't download videos that we already have
# Don't download videos if they're in a channel or playlist (since
# downloading the channel/playlist downloads the videos it contains)
# (Exception: download a single video if that's what the calling code
# has specifically requested)
if isinstance(media_data_obj, media.Video):
if media_data_obj.dl_flag \
or (
not isinstance(media_data_obj.parent_obj, media.Folder) \
and not init_flag
):
return
# Don't create a download.DownloadItem object for a media.Folder,
# obviously
if not isinstance(media_data_obj, media.Folder):
# Create a new download.DownloadItem object...
self.download_item_count += 1
download_item_obj = DownloadItem(
self.download_item_count,
media_data_obj,
options_manager_obj,
)
# ...and add it to our list
self.download_item_list.append(download_item_obj.dbid)
self.download_item_dict[download_item_obj.dbid] \
= download_item_obj
# If the media data object has children, call this function recursively
# for each of them
if not isinstance(media_data_obj, media.Video):
for child_obj in media_data_obj.child_list:
self.create_item(child_obj)
@synchronise(_SYNC_LOCK)
def fetch_next_item(self):
"""Called by downloads.DownloadManager.run().
Based on DownloadList.fetch_next().
Returns:
The next downloads.DownloadItem object, or None if there are none
left.
"""
for dbid in self.download_item_list:
this_item = self.download_item_dict[dbid]
# Don't return an item that's marked as constants.MAIN_STAGE_ACTIVE
if this_item.stage == constants.MAIN_STAGE_QUEUED:
return this_item
return None
def get_options_manager(self, media_data_obj):
"""Called by self.create_item() or by this function recursively.
Fetches the options.OptionsManager which applies to the specified media
data object.
The media data object might specify its own options.OptionsManager, or
we might have to use the parent's, or the parent's parent's (and so
on). As a last resort, use General Options Manager.
Args:
obj(media.Video, media.Channel, media.Playlist, media.Folder):
A media data object
Returns:
The options.OptionsManager object that applies to the specified
media data object
"""
if media_data_obj.options_obj:
return media_data_obj.options_obj
elif media_data_obj.parent_obj:
return self.get_options_manager(media_data_obj.parent_obj)
else:
return self.app_obj.general_options_obj
class DownloadItem(object):
"""Called by downloads.DownloadList.create_item().
Based on the DownloadItem class in youtube-dl-gui.
Python class used to track the download status of a media data object
(media.Video, media.Channel, media.Playlist or media.Folder), one of many
in a downloads.DownloadList object.
Args:
dbid (int) - The number of downloads.DownloadItem objects created, used
to give each one a unique ID
media_data_obj (media.Video, media.Channel, media.Playlist,
media.Folder): A media data object to be downloaded
options_manager_obj (options.OptionsManager): The object which
specifies download options for the media data object
"""
# Standard class methods
def __init__(self, dbid, media_data_obj, options_manager_obj):
# IV list - class objects
# -----------------------
# The media data object to be downloaded
self.media_data_obj = media_data_obj
# The object which specifies download options for the media data object
self.options_manager_obj = options_manager_obj
# IV list - other
# ---------------
# A unique ID for this object
self.dbid = dbid
# The current download stage
self.stage = constants.MAIN_STAGE_QUEUED
class VideoDownloader(object):
"""Called by downloads.DownloadWorker.run().
Based on the YoutubeDLDownloader class in youtube-dl-gui.
Python class to create a system child process. Uses the child process to
instruct youtube-dl to download all videos associated with the URL
described by a downloads.DownloadItem object (which might be an individual
video, or a channel or playlist).
Reads from the child process STDOUT and STDERR, having set up a
downloads.PipeReader object to do so in an asynchronous way.
Sets self.return_code to a value in the range 0-5, described below. The
parent downloads.DownloadWorker object checks that return code once this
object's child process has finished.
Args:
download_manager_obj (downloads.DownloadManager) - The download
manager object handling the entire download operation.
download_worker_obj (downloads.DownloadWorker) - The parent download
worker object. The download manager uses multiple workers to
implement simultaneous downloads. The download manager checks for
free workers and, when it finds one, assigns it a
download.DownloadItem object. When the worker is assigned a
download item, it creates a new instance of this object to
interface with youtube-dl, and waits for this object to return a
return code.
download_item_obj (downloads.DownloadItem) - The download item object
describing the URL from which youtube-dl should download video(s).
Warnings:
The calling function is responsible for calling the close() method
when it's finished with this object, in order for this object to
properly close down.
"""
# Attributes
# Valid values for self.return_code. The larger the number, the higher in
# the hierarchy of return codes.
# Codes lower in the hierarchy (with a smaller number) cannot overwrite
# higher in the hierarchy (with a bigger number)
#
# 0 - The download operation completed successfully
OK = 0
# 1 - A warning occured during the download operation
WARNING = 1
# 2 - An error occured during the download operation
ERROR = 2
# 3 - The corresponding url video file was larger or smaller from the given
# filesize limit
FILESIZE_ABORT = 3
# 4 - The video(s) for the specified URL have already been downloaded
ALREADY = 4
# 5 - The download operation was stopped by the user
STOPPED = 5
# Standard class methods
def __init__(self, download_manager_obj, download_worker_obj, \
download_item_obj):
# IV list - class objects
# -----------------------
# The downloads.DownloadManager object handling the entire download
# operation
self.download_manager_obj = download_manager_obj
# The parent downloads.DownloadWorker object
self.download_worker_obj = download_worker_obj
# The downloads.DownloadItem object describing the URL from which
# youtube-dl should download video(s)
self.download_item_obj = download_item_obj
# This object reads from the child process STDOUT and STDERR in an
# asynchronous way
# Standard Python synchronised queue classes
self.stdout_queue = queue.Queue()
self.stderr_queue = queue.Queue()
# The downloads.PipeReader objects created to handle reading from the
# pipes
self.stdout_reader = PipeReader(self.stdout_queue)
self.stderr_reader = PipeReader(self.stderr_queue)
# The child process created by self.create_child_process()
self.child_process = None
# IV list - other
# ---------------
# The current return code, using values in the range 0-5, as described
# above
# The value remains set to self.OK unless we encounter any problems
# The larger the number, the higher in the hierarchy of return codes.
# Codes lower in the hierarchy (with a smaller number) cannot
# overwrite higher in the hierarchy (with a bigger number)
self.return_code = self.OK
# Flag set to True if we are simulating downloads for this media data
# object, or False if we actually downloading videos (set below)
self.dl_sim_flag = None
# youtube-dl is passed a URL, which might represent an individual
# video, a channel or a playlist
# Assume it's an individual video unless youtube-dl reports a
# channel or playlist (in which case, we can update these IVs later)
# For simulated downloads, both IVs are set to the number of
# videos actually found
self.video_num = None
self.video_total = None
# self.extract_stdout_data() detects the completion of a download job
# in one of several ways
# The first time it happens for each individual video,
# self.extract_stdout_data() takes action. It calls
# self.confirm_new_video(), self.confirm_old_video() or
# self.confirm_sim_video() when required
# On subsequent occasions, the completion message is ignored (as
# youtube-dl may pass us more than one completion message for a
# single video)
# Dictionary of videos, used to check for the first completion message
# for each unique video
# Dictionary in the form
# key = the video number (matches self.video_num)
# value = the video name (not actually used by anything at the
# moment)
self.video_check_dict = {}
# Code
# ----
# Initialise IVs depending on whether this is a real or simulated
# download
media_data_obj = self.download_item_obj.media_data_obj
# If the media data object is a video, channel or playlist, it can be
# marked as a simulated download only
# If it's a video inside a folder and the folder itself is marked as
# simulated downloads only, apply that to all videos in the folder
if self.download_manager_obj.force_sim_flag \
or media_data_obj.dl_sim_flag \
or (
isinstance(media_data_obj, media.Video) \
and isinstance(media_data_obj.parent_obj, media.Folder) \
and media_data_obj.parent_obj.dl_sim_flag
):
self.dl_sim_flag = True
self.video_num = 0
self.video_total = 0
else:
self.dl_sim_flag = False
self.video_num = 1
self.video_total = 1
# Public class methods
def OLDdo_download(self):
"""Called by downloads.DownloadWorker.run().
Based on YoutubeDLDownloader.download().
Downloads video(s) from a URL described by self.download_item_obj.
Returns:
The final return code, a value in the range 0-5 (as described
above)
"""
# Import the main application (for convenience)
app_obj = self.download_manager_obj.app_obj
# Set the default return code. Everything is OK unless we encounter
# any problems
self.return_code = self.OK
# Reset the errors/warnings stored in the media data object, the last
# time it was checked/downloaded
self.download_item_obj.media_data_obj.reset_error_warning()
# Prepare a system command...
cmd_list = self.get_system_cmd()
# ...and create a new child process using that command
self.create_child_process(cmd_list)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout,
)
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr,
)
# While downloading the video, update the callback function with
# the status of the current job
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
stdout = utils.convert_item(stdout, to_unicode=True)
if stdout:
# Convert the statistics into a python dictionary in a
# standard format, specified in the comments for
# self.extract_stdout_data()
dl_stat_dict = self.extract_stdout_data(stdout)
# If the job's status is constants.COMPLETED_STAGE_ALREADY
# or constants.ERROR_STAGE_ABORT, set our
# self.return_code IV
self.extract_stdout_status(dl_stat_dict)
# Pass the dictionary on to self.download_worker_obj so the
# main window can be updated
self.download_worker_obj.data_callback(dl_stat_dict)
if (app_obj.ytdl_write_stdout_flag):
print(stdout)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
stderr = utils.convert_item(stderr, to_unicode=True)
if self.is_warning(stderr):
self.set_return_code(self.WARNING)
self.download_item_obj.media_data_obj.set_warning(stderr)
else:
self.set_return_code(self.ERROR)
self.download_item_obj.media_data_obj.set_error(stderr)
if (app_obj.ytdl_write_stderr_flag):
print(stderr)
# We also set the return code to self.ERROR if the download didn't
# start or if the child process return code is greater than 0
# Original notes from youtube-dl-gui:
# NOTE: In Linux if the called script is just empty Python exits
# normally (ret=0), so we cant detect this or similar cases
# using the code below
# NOTE: In Unix a negative return code (-N) indicates that the child
# was terminated by signal N (e.g. -9 = SIGKILL)
if self.child_process is None:
self.set_return_code(self.ERROR)
self.download_item_obj.media_data_obj.set_error(
'Download did not start',
)
elif self.child_process.returncode > 0:
self.set_return_code(self.ERROR)
self.download_item_obj.media_data_obj.set_error(
'Child process exited with non-zero code: {}'.format(
self.child_process.returncode,
)
)
# Pass a dictionary of values to downloads.DownloadWorker, confirming
# the result of the job. The values are passed on to the main
# window
self.last_data_callback()
# Pass the result back to the parent downloads.DownloadWorker object
return self.return_code
def do_download(self):
"""Called by downloads.DownloadWorker.run().
Based on YoutubeDLDownloader.download().
Downloads video(s) from a URL described by self.download_item_obj.
Returns:
The final return code, a value in the range 0-5 (as described
above)
"""
# Import the main application (for convenience)
app_obj = self.download_manager_obj.app_obj
# Set the default return code. Everything is OK unless we encounter
# any problems
self.return_code = self.OK
# Reset the errors/warnings stored in the media data object, the last
# time it was checked/downloaded
self.download_item_obj.media_data_obj.reset_error_warning()
# Prepare a system command...
cmd_list = self.get_system_cmd()
# ...and create a new child process using that command
self.create_child_process(cmd_list)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout,
)
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr,
)
# While downloading the video, update the callback function with
# the status of the current job
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
# # (Convert Python2 to Python3)
# stdout = self.stdout_queue.get_nowait().rstrip()
# stdout = utils.convert_item(stdout, to_unicode=True)
stdout = self.stdout_queue.get_nowait().rstrip().decode('utf-8')
if stdout:
# Convert the statistics into a python dictionary in a
# standard format, specified in the comments for
# self.extract_stdout_data()
dl_stat_dict = self.extract_stdout_data(stdout)
# If the job's status is constants.COMPLETED_STAGE_ALREADY
# or constants.ERROR_STAGE_ABORT, set our
# self.return_code IV
self.extract_stdout_status(dl_stat_dict)
# Pass the dictionary on to self.download_worker_obj so the
# main window can be updated
self.download_worker_obj.data_callback(dl_stat_dict)
if (app_obj.ytdl_write_stdout_flag):
print(stdout)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
# # (Convert Python2 to Python3)
# stderr = self.stderr_queue.get_nowait().rstrip()
# stderr = utils.convert_item(stderr, to_unicode=True)
stderr = self.stderr_queue.get_nowait().rstrip().decode('utf-8')
if self.is_warning(stderr):
self.set_return_code(self.WARNING)
self.download_item_obj.media_data_obj.set_warning(stderr)
else:
self.set_return_code(self.ERROR)
self.download_item_obj.media_data_obj.set_error(stderr)
if (app_obj.ytdl_write_stderr_flag):
print(stderr)
# We also set the return code to self.ERROR if the download didn't
# start or if the child process return code is greater than 0
# Original notes from youtube-dl-gui:
# NOTE: In Linux if the called script is just empty Python exits
# normally (ret=0), so we cant detect this or similar cases
# using the code below
# NOTE: In Unix a negative return code (-N) indicates that the child
# was terminated by signal N (e.g. -9 = SIGKILL)
if self.child_process is None:
self.set_return_code(self.ERROR)
self.download_item_obj.media_data_obj.set_error(
'Download did not start',
)
elif self.child_process.returncode > 0:
self.set_return_code(self.ERROR)
self.download_item_obj.media_data_obj.set_error(
'Child process exited with non-zero code: {}'.format(
self.child_process.returncode,
)
)
# Pass a dictionary of values to downloads.DownloadWorker, confirming
# the result of the job. The values are passed on to the main
# window
self.last_data_callback()
# Pass the result back to the parent downloads.DownloadWorker object
return self.return_code
def close(self):
"""Called by downloads.DownloadWorker.run() and .close().
Destructor function for this object.
"""
# Tell the PipeReader objects to shut down, thus joining their threads
self.stdout_reader.join()
self.stderr_reader.join()
def confirm_new_video(self, dir_path, filename, extension):
"""Called by self.extract_stdout_data().
A successful download is announced in one of several ways.
When an announcement is detected, this function is called. Use the
first announcement to update self.video_check_dict, and ignore
subsequent announcements.
Args:
dir_path (string): The full path to the directory in which the
video is saved, e.g. '/home/yourname/tartube/downloads/Videos'
filename (string): The video's filename, e.g. 'My Video'
extension (string): The video's extension, e.g. '.mp4'
"""
if not self.video_num in self.video_check_dict:
self.video_check_dict[self.video_num] = filename
# Create a new media.Video object for the video
app_obj = self.download_manager_obj.app_obj
video_obj = app_obj.create_video_from_download(
self.download_item_obj,
dir_path,
filename,
extension,
)
# If downloading from a playlist, remember the video's index in
# that playlist
if isinstance(video_obj.parent_obj, media.Playlist):
video_obj.set_index(self.video_num)
# Fetch the options.OptionsManager object used for this download
options_manager_obj = self.download_worker_obj.options_manager_obj
# Update the main window
app_obj.announce_video_download(
self.download_item_obj,
video_obj,
options_manager_obj.options_dict['keep_description'],
options_manager_obj.options_dict['keep_info'],
options_manager_obj.options_dict['keep_thumbnail'],
)
def confirm_old_video(self, dir_path, filename, extension):
"""Called by self.extract_stdout_data().
When youtube-dl reports a video has already been downloaded, make sure
the media.Video object is marked as downloaded, and upate the video
catalogue in the main window if necessary.
Args:
dir_path (string): The full path to the directory in which the
video is saved, e.g. '/home/yourname/tartube/downloads/Videos'
filename (string): The video's filename, e.g. 'My Video'
extension (string): The video's extension, e.g. '.mp4'
"""
# Create shortcut variables (for convenience)
app_obj = self.download_manager_obj.app_obj
media_data_obj = self.download_item_obj.media_data_obj
if isinstance(media_data_obj, media.Video):
if not media_data_obj.dl_flag:
media_data_obj.set_dl_flag(True)
app_obj.main_win_obj.video_catalogue_update_row(media_data_obj)
else:
# media_data_obj is a media.Channel or media.Playlist object. Check
# its child objects, looking for a matching video
match_obj = media_data_obj.find_matching_video(app_obj, filename)
if match_obj:
match_obj.set_dl_flag(True)
app_obj.main_win_obj.video_catalogue_update_row(match_obj)
else:
# No match found, so create a new media.Video object for the
# video file that already exists on the user's filesystem
self.video_check_dict[self.video_num] = filename
video_obj = app_obj.create_video_from_download(
self.download_item_obj,
dir_path,
filename,
extension,
)
# Fetch the options.OptionsManager object used for this
# download
options_manager_obj \
= self.download_worker_obj.options_manager_obj
# Update the main window
app_obj.announce_video_download(
self.download_item_obj,
video_obj,
options_manager_obj.options_dict['keep_description'],
options_manager_obj.options_dict['keep_info'],
options_manager_obj.options_dict['keep_thumbnail'],
)
def OLDconfirm_sim_video(self, json_dict):
"""Called by self.extract_stdout_data().
After a successful simulated download, youtube-dl presents us with JSON
data for the video. Use that data to update everything.
Args:
json_dict (dict): JSON data from STDOUT, converted into a python
dictionary
"""
# IMport the main application (for convenience)
app_obj = self.download_manager_obj.app_obj
# From the JSON dictionary, extract the data we need
if '_filename' in json_dict:
full_path = json_dict['_filename']
path, filename, extension = self.extract_filename(full_path)
else:
return app_obj.system_error(
302,
'Missing filename in JSON data',
)
if 'upload_date' in json_dict:
# date_string in form YYYYMMDD
date_string = json_dict['upload_date']
dt_obj = datetime.datetime.strptime(date_string, '%Y%m%d')
upload_time = dt_obj.strftime('%s')
else:
upload_time = None
if 'duration' in json_dict:
duration = json_dict['duration']
else:
duration = None
if 'title' in json_dict:
name = json_dict['title']
else:
name = None
if 'description' in json_dict:
descrip = json_dict['description']
else:
descrip = None
if 'thumbnail' in json_dict:
thumbnail = json_dict['thumbnail']
else:
thumbnail = None
if 'webpage_url' in json_dict:
source = json_dict['webpage_url']
else:
source = None
if 'playlist_index' in json_dict:
playlist_index = json_dict['playlist_index']
else:
playlist_index = None
# Does an existing media.Video object match this video?
media_data_obj = self.download_item_obj.media_data_obj
video_obj = None
if isinstance(media_data_obj, media.Video):
video_obj = media_data_obj
else:
# media_data_obj is a media.Channel or media.Playlist object. Check
# its child objects, looking for a matching video
# (video_obj is set to None, if no match is found)
video_obj = media_data_obj.find_matching_video(app_obj, filename)
if not video_obj:
# No matching media.Video object found, so create a new one
video_obj = app_obj.create_video_from_download(
self.download_item_obj,
path,
filename,
extension,
)
# Update its IVs with the JSON information we extracted
if name is not None:
video_obj.set_name(name)
if upload_time is not None:
video_obj.set_upload_time(upload_time)
if duration is not None:
video_obj.set_duration(duration)
if source is not None:
video_obj.set_source(source)
if descrip is not None:
video_obj.set_video_descrip(
descrip,
app_obj.main_win_obj.long_string_max_len,
)
# Only save the playlist index when this video is actually stored
# inside a media.Playlist object
if isinstance(video_obj.parent_obj, media.Playlist) \
and playlist_index is not None:
video_obj.set_index(playlist_index)
else:
# If the 'Add videos' button was used, the path/filename/extension
# won't be set yet
if not video_obj.file_dir and full_path:
video_obj.set_file(path, filename, extension)
# Update any video object IVs that are not set
if video_obj.name == app_obj.default_video_name \
and name is not None:
video_obj.set_name(name)
if not video_obj.upload_time and upload_time is not None:
video_obj.set_upload_time(upload_time)
if not video_obj.duration and duration is not None:
video_obj.set_duration(duration)
if not video_obj.source and source is not None:
video_obj.set_source(source)
if not video_obj.descrip and descrip is not None:
video_obj.set_video_descrip(
descrip,
app_obj.main_win_obj.long_string_max_len,
)
# Only save the playlist index when this video is actually stored
# inside a media.Playlist object
if not video_obj.index \
and isinstance(video_obj.parent_obj, media.Playlist) \
and playlist_index is not None:
video_obj.set_index(playlist_index)
# Deal with the video description, JSON data and thumbnail, according
# to the settings in options.OptionsManager
options_dict =self.download_worker_obj.options_manager_obj.options_dict
if descrip and options_dict['write_description']:
descrip_path = os.path.join(path, filename + '.description')
if not options_dict['sim_keep_description']:
descrip_path = utils.convert_path_to_temp(
app_obj,
descrip_path,
)
# (Don't replace a file that already exists)
if not os.path.isfile(descrip_path):
fh = open(descrip_path, 'w')
fh.write(descrip.encode('utf-8'))
fh.close()
if options_dict['write_info']:
json_path = os.path.join(path, filename + '.info.json')
if not options_dict['sim_keep_info']:
json_path = utils.convert_path_to_temp(app_obj, json_path)
if not os.path.isfile(json_path):
with open(json_path, 'w') as outfile:
json.dump(json_dict, outfile, indent=4)
if thumbnail and options_dict['write_thumbnail']:
# Download the thumbnail, if we don't already have it
# The thumbnail's URL is something like
# 'https://i.ytimg.com/vi/abcdefgh/maxresdefault.jpg'
# When saved to disc by youtube-dl, the file is given the same name
# as the video (but with a different extension)
# Get the thumbnail's extension...
remote_file, remote_ext = os.path.splitext(thumbnail)
# ...and thus get the filename used by youtube-dl when storing the
# thumbnail locally
thumb_path = os.path.join(
video_obj.file_dir,
video_obj.file_name + remote_ext,
)
if not options_dict['sim_keep_thumbnail']:
thumb_path = utils.convert_path_to_temp(app_obj, thumb_path)
if not os.path.isfile(thumb_path):
request_obj = requests.get(thumbnail)
with open(thumb_path, 'wb') as outfile:
outfile.write(request_obj.content)
# Update the main window
app_obj.announce_video_download(self.download_item_obj, video_obj)
def confirm_sim_video(self, json_dict):
"""Called by self.extract_stdout_data().
After a successful simulated download, youtube-dl presents us with JSON
data for the video. Use that data to update everything.
Args:
json_dict (dict): JSON data from STDOUT, converted into a python
dictionary
"""
# IMport the main application (for convenience)
app_obj = self.download_manager_obj.app_obj
# From the JSON dictionary, extract the data we need
if '_filename' in json_dict:
full_path = json_dict['_filename']
path, filename, extension = self.extract_filename(full_path)
else:
return app_obj.system_error(
302,
'Missing filename in JSON data',
)
if 'upload_date' in json_dict:
# date_string in form YYYYMMDD
date_string = json_dict['upload_date']
dt_obj = datetime.datetime.strptime(date_string, '%Y%m%d')
upload_time = dt_obj.strftime('%s')
else:
upload_time = None
if 'duration' in json_dict:
duration = json_dict['duration']
else:
duration = None
if 'title' in json_dict:
name = json_dict['title']
else:
name = None
if 'description' in json_dict:
descrip = json_dict['description']
else:
descrip = None
if 'thumbnail' in json_dict:
thumbnail = json_dict['thumbnail']
else:
thumbnail = None
if 'webpage_url' in json_dict:
source = json_dict['webpage_url']
else:
source = None
if 'playlist_index' in json_dict:
playlist_index = json_dict['playlist_index']
else:
playlist_index = None
# Does an existing media.Video object match this video?
media_data_obj = self.download_item_obj.media_data_obj
video_obj = None
if isinstance(media_data_obj, media.Video):
video_obj = media_data_obj
else:
# media_data_obj is a media.Channel or media.Playlist object. Check
# its child objects, looking for a matching video
# (video_obj is set to None, if no match is found)
video_obj = media_data_obj.find_matching_video(app_obj, filename)
if not video_obj:
# No matching media.Video object found, so create a new one
video_obj = app_obj.create_video_from_download(
self.download_item_obj,
path,
filename,
extension,
)
# Update its IVs with the JSON information we extracted
if name is not None:
video_obj.set_name(name)
if upload_time is not None:
video_obj.set_upload_time(upload_time)
if duration is not None:
video_obj.set_duration(duration)
if source is not None:
video_obj.set_source(source)
if descrip is not None:
video_obj.set_video_descrip(
descrip,
app_obj.main_win_obj.long_string_max_len,
)
# Only save the playlist index when this video is actually stored
# inside a media.Playlist object
if isinstance(video_obj.parent_obj, media.Playlist) \
and playlist_index is not None:
video_obj.set_index(playlist_index)
else:
# If the 'Add videos' button was used, the path/filename/extension
# won't be set yet
if not video_obj.file_dir and full_path:
video_obj.set_file(path, filename, extension)
# Update any video object IVs that are not set
if video_obj.name == app_obj.default_video_name \
and name is not None:
video_obj.set_name(name)
if not video_obj.upload_time and upload_time is not None:
video_obj.set_upload_time(upload_time)
if not video_obj.duration and duration is not None:
video_obj.set_duration(duration)
if not video_obj.source and source is not None:
video_obj.set_source(source)
if not video_obj.descrip and descrip is not None:
video_obj.set_video_descrip(
descrip,
app_obj.main_win_obj.long_string_max_len,
)
# Only save the playlist index when this video is actually stored
# inside a media.Playlist object
if not video_obj.index \
and isinstance(video_obj.parent_obj, media.Playlist) \
and playlist_index is not None:
video_obj.set_index(playlist_index)
# Deal with the video description, JSON data and thumbnail, according
# to the settings in options.OptionsManager
options_dict =self.download_worker_obj.options_manager_obj.options_dict
if descrip and options_dict['write_description']:
descrip_path = os.path.join(path, filename + '.description')
if not options_dict['sim_keep_description']:
descrip_path = utils.convert_path_to_temp(
app_obj,
descrip_path,
)
# (Don't replace a file that already exists)
if not os.path.isfile(descrip_path):
# # Convert Python2 to Python3
# fh = open(descrip_path, 'w')
fh = open(descrip_path, 'wb')
fh.write(descrip.encode('utf-8'))
fh.close()
if options_dict['write_info']:
json_path = os.path.join(path, filename + '.info.json')
if not options_dict['sim_keep_info']:
json_path = utils.convert_path_to_temp(app_obj, json_path)
if not os.path.isfile(json_path):
with open(json_path, 'w') as outfile:
json.dump(json_dict, outfile, indent=4)
if thumbnail and options_dict['write_thumbnail']:
# Download the thumbnail, if we don't already have it
# The thumbnail's URL is something like
# 'https://i.ytimg.com/vi/abcdefgh/maxresdefault.jpg'
# When saved to disc by youtube-dl, the file is given the same name
# as the video (but with a different extension)
# Get the thumbnail's extension...
remote_file, remote_ext = os.path.splitext(thumbnail)
# ...and thus get the filename used by youtube-dl when storing the
# thumbnail locally
thumb_path = os.path.join(
video_obj.file_dir,
video_obj.file_name + remote_ext,
)
if not options_dict['sim_keep_thumbnail']:
thumb_path = utils.convert_path_to_temp(app_obj, thumb_path)
if not os.path.isfile(thumb_path):
request_obj = requests.get(thumbnail)
with open(thumb_path, 'wb') as outfile:
outfile.write(request_obj.content)
# Update the main window
app_obj.announce_video_download(self.download_item_obj, video_obj)
def create_child_process(self, cmd_list):
"""Called by self.do_download() immediately after the call to
self.get_system_cmd().
Based on YoutubeDLDownloader._create_process().
Executes the system command, creating a new child process which
executes youtube-dl.
Args:
cmd_list (list): Python list that contains the command to execute.
Returns:
None on success, or the new value of self.return_code if there's an
error
"""
info = preexec = None
if os.name == 'nt':
# Hide the child process window that MS Windows helpfully creates
# for us
info = subprocess.STARTUPINFO()
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
else:
# Make this child process the process group leader, so that we can
# later kill the whole process group with os.killpg
preexec = os.setsid
# Encode the system command for the child process, converting unicode
# to str so the MS Windows shell can accept it (see
# http://stackoverflow.com/a/9951851/35070 )
if sys.version_info < (3, 0):
cmd_list = utils.convert_item(cmd_list, to_unicode=False)
try:
self.child_process = subprocess.Popen(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=preexec,
startupinfo=info,
)
except (ValueError, OSError) as error:
# (There is no need to update the media data object's error list,
# as the code in self.do_download() will notice the child
# process didn't start, and set its own error message)
self.set_return_code(self.ERROR)
def extract_filename(self, input_data):
"""Called by self.extract_stdout_data().
Based on the extract_data() function in youtube-dl-gui's
downloaders.py.
Extracts various components of a filename.
Args:
input_data (string): Full path to a file which has been downloaded
and saved to the filesystem
Returns:
Returns the path, filename and extension components of the full
file path.
"""
path, fullname = os.path.split(input_data.strip("\""))
filename, extension = os.path.splitext(fullname)
return path, filename, extension
def extract_stdout_data(self, stdout):
"""Called by self.do_download().
Based on the extract_data() function in youtube-dl-gui's
downloaders.py.
Extracts youtube-dl statistics from the child process.
Args:
stdout (string): String that contains a line from the child process
STDOUT (i.e., a message from youtube-dl)
Returns:
Python dictionary in a standard format also used by the main window
code. Dictionaries in this format are generally called
'dl_stat_dict' (or some variation of it).
The returned dictionary can be empty if there is no data to
extract, otherwise it contains one or more of the following keys:
'status' : Contains the status of the download
'path' : Destination path
'filename' : The filename without the extension
'extension' : The file extension
'percent' : The percentage of the video being downloaded
'eta' : Estimated time for the completion of the
download
'speed' : Download speed
'filesize' : The size of the video file being downloaded
'playlist_index' : The playlist index of the current video file
being downloaded
'playlist_size' : The number of videos in the playlist
'dl_sim_flag' : Flag set to True if we are simulating downloads
for this media data object, or False if we
actually downloading videos (set below)
"""
# Initialise the dictionary with default key-value pairs for the main
# window to display, to be overwritten (if possible) with new key-
# value pairs as this function interprets the STDOUT message
dl_stat_dict = {
'playlist_index': self.video_num,
'playlist_size': self.video_total,
'dl_sim_flag': self.dl_sim_flag,
}
# If STDOUT has not been received by this function, then the main
# window can be passed just the default key-value pairs
if not stdout:
return dl_stat_dict
# In some cases, we want to preserve the multiple successive whitespace
# characters in the STDOUT message, in order to extract filenames
# in their original form
# In other cases, we just eliminate multiple successive whitespace
# characters
stdout_with_spaces_list = stdout.split(' ')
stdout_list = stdout.split()
# Extract the data
stdout_list[0] = stdout_list[0].lstrip('\r')
if stdout_list[0] == '[download]':
dl_stat_dict['status'] = constants.ACTIVE_STAGE_DOWNLOAD
# Get path, filename and extension
if stdout_list[1] == 'Destination:':
path, filename, extension = self.extract_filename(
' '.join(stdout_with_spaces_list[2:]),
)
# ('filename' is something like 'My video.mpetg.f135', which
# is not what we want, so don't update self.video_check_dict)
dl_stat_dict['path'] = path
dl_stat_dict['filename'] = filename
dl_stat_dict['extension'] = extension
# Get progress information
if '%' in stdout_list[1]:
if stdout_list[1] == '100%':
dl_stat_dict['percent'] = '100%'
dl_stat_dict['eta'] = ''
dl_stat_dict['speed'] = ''
dl_stat_dict['filesize'] = stdout_list[3]
else:
dl_stat_dict['percent'] = stdout_list[1]
dl_stat_dict['eta'] = stdout_list[7]
dl_stat_dict['speed'] = stdout_list[5]
dl_stat_dict['filesize'] = stdout_list[3]
# Get playlist information (when downloading a channel or a
# playlist, this line is received once per video)
if stdout_list[1] == 'Downloading' and stdout_list[2] == 'video':
dl_stat_dict['playlist_index'] = stdout_list[3]
self.video_num = stdout_list[3]
dl_stat_dict['playlist_size'] = stdout_list[5]
self.video_total = stdout_list[5]
# Remove the 'and merged' part of the STDOUT message when using
# FFmpeg to merge the formats
if stdout_list[-3] == 'downloaded' and stdout_list[-1] == 'merged':
stdout_list = stdout_list[:-2]
stdout_with_spaces_list = stdout_with_spaces_list[:-2]
dl_stat_dict['percent'] = '100%'
# Get file already downloaded status
if stdout_list[-1] == 'downloaded':
dl_stat_dict['status'] = constants.COMPLETED_STAGE_ALREADY
path, filename, extension = self.extract_filename(
' '.join(stdout_with_spaces_list[1:-4]),
)
dl_stat_dict['path'] = path
dl_stat_dict['filename'] = filename
dl_stat_dict['extension'] = extension
self.confirm_old_video(path, filename, extension)
# Get filesize abort status
if stdout_list[-1] == 'Aborting.':
dl_stat_dict['status'] = constants.ERROR_STAGE_ABORT
elif stdout_list[0] == '[hlsnative]':
# Get information from the native HLS extractor (see
# https://github.com/rg3/youtube-dl/blob/master/youtube_dl/
# downloader/hls.py#L54
dl_stat_dict['status'] = constants.ACTIVE_STAGE_DOWNLOAD
if len(stdout_list) == 7:
segment_no = float(stdout_list[6])
current_segment = float(stdout_list[4])
# Get the percentage
percent = '{0:.1f}%'.format(current_segment / segment_no * 100)
dl_stat_dict['percent'] = percent
elif stdout_list[0] == '[ffmpeg]':
# Using FFmpeg, not the the native HLS extractor
# A successful video download is announced in one of several ways.
# Use the first announcement to update self.video_check_dict, and
# ignore subsequent announcements
dl_stat_dict['status'] = constants.ACTIVE_STAGE_POST_PROCESS
# Get the final file extension after the merging process has
# completed
if stdout_list[1] == 'Merging':
path, filename, extension = self.extract_filename(
' '.join(stdout_with_spaces_list[4:]),
)
dl_stat_dict['path'] = path
dl_stat_dict['filename'] = filename
dl_stat_dict['extension'] = extension
self.confirm_new_video(path, filename, extension)
# Get the final file extension after simple FFmpeg post-processing
# (i.e. not after a file merge)
if stdout_list[1] == 'Destination:':
path, filename, extension = self.extract_filename(
' '.join(stdout_with_spaces_list[2:]),
)
dl_stat_dict['path'] = path
dl_stat_dict['filename'] = filename
dl_stat_dict['extension'] = extension
self.confirm_new_video(path, filename, extension)
# Get final file extension after the recoding process
if stdout_list[1] == 'Converting':
path, filename, extension = self.extract_filename(
' '.join(stdout_with_spaces_list[8:]),
)
dl_stat_dict['path'] = path
dl_stat_dict['filename'] = filename
dl_stat_dict['extension'] = extension
self.confirm_new_video(path, filename, extension)
elif stdout_list[0][0] == '{':
# JSON data, the result of a simulated download. Convert to a
# python dictionary
if self.dl_sim_flag:
# (Try/except to check for invalid JSON)
try:
json_dict = json.loads(stdout)
except:
return self.download_manager_obj.app_obj.system_error(
303,
'Invalid JSON data received from server',
)
# (JSON is valid)
self.confirm_sim_video(json_dict)
self.video_num += 1
dl_stat_dict['playlist_index'] = self.video_num
self.video_total += 1
dl_stat_dict['playlist_size'] = self.video_total
dl_stat_dict['status'] = constants.ACTIVE_STAGE_CHECKING
elif stdout_list[0][0] != '[' or stdout_list[0] == '[debug]':
# (Just ignore this output)
return dl_stat_dict
else:
# The download has started
dl_stat_dict['status'] = constants.ACTIVE_STAGE_PRE_PROCESS
return dl_stat_dict
def extract_stdout_status(self, dl_stat_dict):
"""Called by self.do_download() immediately after a call to
self.extract_stdout_data().
Based on YoutubeDLDownloader._extract_info().
If the job's status is constants.COMPLETED_STAGE_ALREADY or
constants.ERROR_STAGE_ABORT, translate that into a new value for the
return code, and then use that value to actually set self.return_code
(which halts the download).
Args:
dl_stat_dict (dict): The Python dictionary returned by the call to
self.extract_stdout_data(), in the standard form described by
the comments for that function
"""
if 'status' in dl_stat_dict:
if dl_stat_dict['status'] == constants.COMPLETED_STAGE_ALREADY:
self.set_return_code(self.ALREADY)
dl_stat_dict['status'] = None
if dl_stat_dict['status'] == constants.ERROR_STAGE_ABORT:
self.set_return_code(self.FILESIZE_ABORT)
dl_stat_dict['status'] = None
def get_system_cmd(self):
"""Called by self.do_download().
Based on YoutubeDLDownloader._get_cmd().
Prepare the system command that creates the child process, executing
youtube-dl.
Returns:
Python list that contains the system command to execute.
"""
options_list = self.download_worker_obj.options_list
# Simulate the download, rather than actually downloading videos, if
# required
if self.dl_sim_flag:
options_list.append('--dump-json')
# Show verbose output (youtube-dl debugging mode), if required
if self.download_manager_obj.app_obj.ytdl_write_verbose_flag:
options_list.append('--verbose')
# Set the list
cmd_list = [self.download_manager_obj.app_obj.ytdl_path] \
+ options_list + [self.download_item_obj.media_data_obj.source]
return cmd_list
def is_child_process_alive(self):
"""Called by self.do_download() and self.stop().
Based on YoutubeDLDownloader._proc_is_alive().
Called continuously during the self.do_download() loop to check whether
the child process has finished or not.
Returns:
True if the child process is alive, otherwise returns False.
"""
if self.child_process is None:
return False
return self.child_process.poll() is None
def is_warning(self, stderr):
"""Called by self.do_download().
Based on YoutubeDLDownloader._is_warning().
After the child process has terminated with an error of some kind,
checks the STERR message to see if it's an error or just a warning.
Args:
stderr (string): A message from the child process STDERR.
Returns:
True if the STDERR message is a warning, False if it's an error
"""
return stderr.split(':')[0] == 'WARNING'
def last_data_callback(self):
"""Called by self.download().
Based on YoutubeDLDownloader._last_data_hook().
After the child process has finished, creates a new Python dictionary
in the standard form described by self.extract_stdout_data().
Sets key-value pairs in the dictonary, then passes it to the parent
downloads.DownloadWorker object, confirming the result of the child
process.
The new key-value pairs are used to update the main window.
"""
dl_stat_dict = {}
if self.return_code == self.OK:
dl_stat_dict['status'] = constants.COMPLETED_STAGE_FINISHED
elif self.return_code == self.ERROR:
dl_stat_dict['status'] = constants.MAIN_STAGE_ERROR
dl_stat_dict['eta'] = ''
dl_stat_dict['speed'] = ''
elif self.return_code == self.WARNING:
dl_stat_dict['status'] = constants.COMPLETED_STAGE_WARNING
dl_stat_dict['eta'] = ''
dl_stat_dict['speed'] = ''
elif self.return_code == self.STOPPED:
dl_stat_dict['status'] = constants.ERROR_STAGE_STOPPED
dl_stat_dict['eta'] = ''
dl_stat_dict['speed'] = ''
elif self.return_code == self.ALREADY:
dl_stat_dict['status'] = constants.COMPLETED_STAGE_ALREADY
else:
dl_stat_dict['status'] = constants.ERROR_STAGE_ABORT
# Use some empty values in dl_stat_dict so that the Progress Tab
# doesn't show arbitrary data from the last file downloaded
dl_stat_dict['filename'] = ''
dl_stat_dict['extension'] = ''
dl_stat_dict['percent'] = ''
dl_stat_dict['eta'] = ''
dl_stat_dict['speed'] = ''
dl_stat_dict['filesize'] = ''
self.download_worker_obj.data_callback(dl_stat_dict)
def set_return_code(self, code):
"""Called by self.do_download() and self.stop().
Based on YoutubeDLDownloader._set_returncode().
After the child process has terminated with an error of some kind,
sets a new value for self.return_code, but only if the new return code
is higher in the hierarchy of return codes than the current value.
Args:
code (int): A return code in the range 0-5
"""
if code >= self.return_code:
self.return_code = code
def stop(self):
"""Called by downloads.DownloadWorker.close().
Terminates the child process and sets this object's return code to
self.STOPPED.
"""
if self.is_child_process_alive():
if os.name == 'nt':
# os.killpg is not available on MS Windows (see
# https://bugs.python.org/issue5115 )
self.child_process.kill()
# When we kill the child process on MS Windows the return code
# gets set to 1, so we want to reset the return code back to
# 0
self.child_process.returncode = 0
else:
os.killpg(self.child_process.pid, signal.SIGKILL)
self.set_return_code(self.STOPPED)
class PipeReader(threading.Thread):
"""Called by downloads.VideoDownloader.__init__().
Based on the PipeReader class in youtube-dl-gui.
Python class used by downloads.VideoDownloader and updates.UpdateManager to
avoid deadlocks when reading from child process pipes STDOUT and STDERR.
This class uses python threads and queues in order to read from child
process pipes in an asynchronous way.
Args:
queue (queue.Queue): Python queue to store the output of the child
process.
Warnings:
All the operations are based on 'str' types. The calling function must
convert the queued items back to 'unicode', if necessary.
"""
# Standard class methods
def __init__(self, queue):
super(PipeReader, self).__init__()
# IV list - other
# ---------------
# Python queue to store the output of the child process.
self.output_queue = queue
# The time (in seconds) between iterations of the loop in self.run()
self.sleep_time = 0.1
# Flag that is set to False by self.join(), which enables the loop in
# self.run() to terminate
self.running_flag = True
# Set by self.attach_file_descriptor(). The file descriptor for the
# child process STDOUT or STDERR
self.file_descriptor = None
# Code
# ----
# Let's get this party started!
self.start()
# Public class methods
def OLDOLDrun(self):
"""Called as a result of self.__init__().
Reads from STDOUT or STERR using the attached filed descriptor.
"""
# Use this flag so that the loop can ignore FFmpeg error messsages
# (because the parent VideoDownloader object shouldn't use that as a
# serious error)
ignore_line = False
while self.running_flag:
if self.file_descriptor is not None:
for line in iter(self.file_descriptor.readline, str('')):
if str('ffmpeg version') in line:
ignore_line = True
if not ignore_line:
self.output_queue.put_nowait(line)
self.file_descriptor = None
ignore_line = False
time.sleep(self.sleep_time)
def run(self):
"""Called as a result of self.__init__().
Reads from STDOUT or STERR using the attached filed descriptor.
"""
# Use this flag so that the loop can ignore FFmpeg error messsages
# (because the parent VideoDownloader object shouldn't use that as a
# serious error)
ignore_line = False
while self.running_flag:
if self.file_descriptor is not None:
# # Convert Python2 to Python3 - the for loop no longer
# # terminates, but produces endless strings instead
# for line in iter(self.file_descriptor.readline, str('')):
for line in iter(self.file_descriptor.readline, str('')):
if line == b'':
break
# # Convert Python2 to Python3
# if str('ffmpeg version') in line:
if str.encode('ffmpeg version') in line:
ignore_line = True
if not ignore_line:
self.output_queue.put_nowait(line)
self.file_descriptor = None
ignore_line = False
time.sleep(self.sleep_time)
def attach_file_descriptor(self, filedesc):
"""Called by downloads.VideoDownloader.do_download().
Sets the file descriptor for the child process STDOUT or STDERR.
Args:
filedesc (filehandle): The open filehandle for STDOUT or STDERR
"""
self.file_descriptor = filedesc
def join(self, timeout=None):
"""Called by downloads.VideoDownloader.close(), which is the destructor
function for that object.
Join the thread and update IVs
Args:
timeout (-): No calling code sets a timeout
"""
self.running_flag = False
super(PipeReader, self).join(timeout)