added download-archive feature (#232)
Justin Mai
Ronan
parent
ebf66a278c
commit
d95a99cc5e
53
README.md
53
README.md
|
|
@ -45,36 +45,40 @@ scdl -l https://soundcloud.com/jumpstreetpsy/low-extender
|
|||
# Download one playlist
|
||||
scdl -l https://soundcloud.com/pandadub/sets/the-lost-ship
|
||||
|
||||
# Download only new tracks from a playlist
|
||||
scdl -l https://soundcloud.com/pandadub/sets/the-lost-ship --download-archive archive.txt
|
||||
|
||||
# Download your likes (with authentification token)
|
||||
scdl me -f
|
||||
```
|
||||
|
||||
### Options:
|
||||
```
|
||||
-h --help Show this screen
|
||||
--version Show version
|
||||
me Use the user profile from the auth_token
|
||||
-l [url] URL can be track/playlist/user
|
||||
-s Download the stream of a user (token needed)
|
||||
-a Download all tracks of a user (including reposts)
|
||||
-t Download all uploads of a user (no reposts)
|
||||
-f Download all favorites of a user
|
||||
-C Download all commented by a user
|
||||
-p Download all playlists of a user
|
||||
-m Download all liked and owned playlists of a user
|
||||
-c Continue if a downloaded file already exists
|
||||
-o [offset] Begin with a custom offset
|
||||
--path [path] Use a custom path for downloaded files
|
||||
--min-size [min-size] Skip tracks smaller than size (k/m/g)
|
||||
--max-size [max-size] Skip tracks larger than size (k/m/g)
|
||||
--hidewarnings Hide Warnings. (use with precaution)
|
||||
--addtofile Add the artist name to the filename if it isn't in the filename already
|
||||
--addtimestamp Adds the timestamp of the creation of the track to the title (useful to sort chronologically)
|
||||
--onlymp3 Download only the mp3 file even if the track is Downloadable
|
||||
--error Set log level to ERROR
|
||||
--debug Set log level to DEBUG
|
||||
--hide-progress Hide the wget progress bar
|
||||
--no-playlist-folder Download playlist tracks into directory, instead of making a playlist subfolder (the default)
|
||||
-h --help Show this screen
|
||||
--version Show version
|
||||
me Use the user profile from the auth_token
|
||||
-l [url] URL can be track/playlist/user
|
||||
-s Download the stream of a user (token needed)
|
||||
-a Download all tracks of a user (including reposts)
|
||||
-t Download all uploads of a user (no reposts)
|
||||
-f Download all favorites of a user
|
||||
-C Download all commented by a user
|
||||
-p Download all playlists of a user
|
||||
-m Download all liked and owned playlists of a user
|
||||
-c Continue if a downloaded file already exists
|
||||
-o [offset] Begin with a custom offset
|
||||
--path [path] Use a custom path for downloaded files
|
||||
--min-size [min-size] Skip tracks smaller than size (k/m/g)
|
||||
--max-size [max-size] Skip tracks larger than size (k/m/g)
|
||||
--hidewarnings Hide Warnings. (use with precaution)
|
||||
--addtofile Add the artist name to the filename if it isn't in the filename already
|
||||
--addtimestamp Adds the timestamp of the creation of the track to the title (useful to sort chronologically)
|
||||
--onlymp3 Download only the mp3 file even if the track is Downloadable
|
||||
--error Set log level to ERROR
|
||||
--debug Set log level to DEBUG
|
||||
--hide-progress Hide the wget progress bar
|
||||
--no-playlist-folder Download playlist tracks into directory, instead of making a playlist subfolder (the default)
|
||||
--download-archive [file] Keep track of track IDs in an archive file and skip already-downloaded files
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -85,6 +89,7 @@ scdl me -f
|
|||
* Download all songs from one playlist
|
||||
* Download all songs from all playlists from a user
|
||||
* Download all songs from a user's favorites
|
||||
* Download only new tracks from a list (playlist, favorites, etc.)
|
||||
* Set the tags with mutagen (Title / Artist / Album / Artwork)
|
||||
* Create playlist files when downloading a playlist
|
||||
|
||||
|
|
|
|||
176
scdl/scdl.py
176
scdl/scdl.py
|
|
@ -6,10 +6,10 @@
|
|||
Usage:
|
||||
scdl -l <track_url> [-a | -f | -C | -t | -p][-c][-o <offset>]\
|
||||
[--hidewarnings][--debug | --error][--path <path>][--addtofile][--addtimestamp][--onlymp3]
|
||||
[--hide-progress][--min-size <size>][--max-size <size>][--remove][--no-playlist-folder]
|
||||
[--hide-progress][--min-size <size>][--max-size <size>][--remove][--no-playlist-folder][--download-archive <file>]
|
||||
scdl me (-s | -a | -f | -t | -p | -m)[-c][-o <offset>]\
|
||||
[--hidewarnings][--debug | --error][--path <path>][--addtofile][--addtimestamp][--onlymp3]
|
||||
[--hide-progress][--min-size <size>][--max-size <size>][--remove][--no-playlist-folder]
|
||||
[--hide-progress][--min-size <size>][--max-size <size>][--remove][--no-playlist-folder][--download-archive <file>]
|
||||
scdl -h | --help
|
||||
scdl --version
|
||||
|
||||
|
|
@ -40,6 +40,7 @@ Options:
|
|||
--debug Set log level to DEBUG
|
||||
--hide-progress Hide the wget progress bar
|
||||
--no-playlist-folder Download playlist tracks into directory, instead of making a playlist subfolder (the default)
|
||||
--download-archive [file] Keep track of track IDs in an archive file and skip already-downloaded files
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
|
@ -393,9 +394,11 @@ def try_utime(path, filetime):
|
|||
def get_filename(track, title, is_original = False):
|
||||
invalid_chars = '\/:*?|<>"'
|
||||
username = track['user']['username']
|
||||
|
||||
if arguments['--addtofile']:
|
||||
if username not in title and '-' not in title:
|
||||
title = '{0} - {1}'.format(username, title)
|
||||
logger.debug('Adding "{0}" to filename'.format(username))
|
||||
|
||||
if arguments['--addtimestamp']:
|
||||
# created_at sample: 2017/03/03 09:29:33 +0000
|
||||
|
|
@ -421,15 +424,15 @@ def download_track(track, playlist_name=None, playlist_file=None):
|
|||
|
||||
title = track['title']
|
||||
title = title.encode('utf-8', 'ignore').decode('utf8')
|
||||
if track['streamable']:
|
||||
url = track['stream_url']
|
||||
else:
|
||||
logger.error('{0} is not streamable...'.format(title))
|
||||
return
|
||||
logger.info('Downloading {0}'.format(title))
|
||||
|
||||
# Not streamable
|
||||
if not track['streamable']:
|
||||
logger.error('{0} is not streamable...'.format(title))
|
||||
return
|
||||
|
||||
r = None
|
||||
# filename
|
||||
# Downloadable track
|
||||
if track['downloadable'] and not arguments['--onlymp3']:
|
||||
logger.info('Downloading the original file.')
|
||||
original_url = track['download_url']
|
||||
|
|
@ -446,9 +449,11 @@ def download_track(track, playlist_name=None, playlist_file=None):
|
|||
filename = get_filename(track, title)
|
||||
else:
|
||||
filename = get_filename(track, title)
|
||||
|
||||
|
||||
logger.debug("filename : {0}".format(filename))
|
||||
|
||||
# Skip if file ID or filename already exists
|
||||
if already_downloaded(track, title, filename): return
|
||||
|
||||
# Add the track to the generated m3u playlist file
|
||||
if playlist_file:
|
||||
duration = math.floor(track['duration'] / 1000)
|
||||
|
|
@ -461,64 +466,119 @@ def download_track(track, playlist_name=None, playlist_file=None):
|
|||
if arguments['--remove']:
|
||||
fileToKeep.append(filename)
|
||||
|
||||
# Download
|
||||
if not os.path.isfile(filename):
|
||||
if r is None or r.status_code == 401:
|
||||
r = requests.get(url, params={'client_id': CLIENT_ID}, stream=True)
|
||||
# Streamable track download
|
||||
if r is None or r.status_code == 401:
|
||||
url = track['stream_url']
|
||||
r = requests.get(url, params={'client_id': CLIENT_ID}, stream=True)
|
||||
logger.debug(r.url)
|
||||
if r.status_code == 401 or r.status_code == 429:
|
||||
r = requests.get(url, params={'client_id': ALT_CLIENT_ID}, stream=True)
|
||||
logger.debug(r.url)
|
||||
if r.status_code == 401 or r.status_code == 429:
|
||||
r = requests.get(url, params={'client_id': ALT_CLIENT_ID}, stream=True)
|
||||
logger.debug(r.url)
|
||||
r.raise_for_status()
|
||||
temp = tempfile.NamedTemporaryFile(delete=False)
|
||||
r.raise_for_status()
|
||||
temp = tempfile.NamedTemporaryFile(delete=False)
|
||||
|
||||
total_length = int(r.headers.get('content-length'))
|
||||
total_length = int(r.headers.get('content-length'))
|
||||
|
||||
min_size = arguments.get('--min-size')
|
||||
max_size = arguments.get('--max-size')
|
||||
min_size = arguments.get('--min-size')
|
||||
max_size = arguments.get('--max-size')
|
||||
|
||||
if min_size is not None and total_length < min_size:
|
||||
logging.info('{0} not large enough, skipping'.format(title))
|
||||
return
|
||||
if min_size is not None and total_length < min_size:
|
||||
logging.info('{0} not large enough, skipping'.format(title))
|
||||
return
|
||||
|
||||
if max_size is not None and total_length > max_size:
|
||||
logging.info('{0} too large, skipping'.format(title))
|
||||
return
|
||||
if max_size is not None and total_length > max_size:
|
||||
logging.info('{0} too large, skipping'.format(title))
|
||||
return
|
||||
|
||||
with temp as f:
|
||||
for chunk in progress.bar(
|
||||
r.iter_content(chunk_size=1024),
|
||||
expected_size=(total_length/1024) + 1,
|
||||
hide=True if arguments["--hide-progress"] else False
|
||||
):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
|
||||
shutil.move(temp.name, os.path.join(os.getcwd(), filename))
|
||||
if filename.endswith('.mp3') or filename.endswith('.m4a'):
|
||||
try:
|
||||
set_metadata(track, filename, playlist_name)
|
||||
except Exception as e:
|
||||
logger.error('Error trying to set the tags...')
|
||||
logger.debug(e)
|
||||
else:
|
||||
logger.error("This type of audio doesn't support tagging...")
|
||||
|
||||
#Try to change the real creation date
|
||||
created_at = track['created_at']
|
||||
filetime = int(time.mktime(datetime.strptime(created_at, '%Y/%m/%d %H:%M:%S %z').timetuple()))
|
||||
try_utime(filename,filetime)
|
||||
with temp as f:
|
||||
for chunk in progress.bar(
|
||||
r.iter_content(chunk_size=1024),
|
||||
expected_size=(total_length/1024) + 1,
|
||||
hide=True if arguments["--hide-progress"] else False
|
||||
):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
|
||||
shutil.move(temp.name, os.path.join(os.getcwd(), filename))
|
||||
if filename.endswith('.mp3') or filename.endswith('.m4a'):
|
||||
try:
|
||||
set_metadata(track, filename, playlist_name)
|
||||
except Exception as e:
|
||||
logger.error('Error trying to set the tags...')
|
||||
logger.debug(e)
|
||||
else:
|
||||
if arguments['-c'] or arguments['--remove']:
|
||||
logger.info('{0} already Downloaded'.format(title))
|
||||
return
|
||||
else:
|
||||
logger.error('Music already exists ! (use -c to continue)')
|
||||
sys.exit(0)
|
||||
logger.error("This type of audio doesn't support tagging...")
|
||||
|
||||
#Try to change the real creation date
|
||||
created_at = track['created_at']
|
||||
filetime = int(time.mktime(datetime.strptime(created_at, '%Y/%m/%d %H:%M:%S %z').timetuple()))
|
||||
try_utime(filename,filetime)
|
||||
|
||||
logger.info('{0} Downloaded.\n'.format(filename))
|
||||
record_download_archive(track)
|
||||
|
||||
|
||||
def already_downloaded(track, title, filename=None):
|
||||
"""
|
||||
Returns True if the file has already been downloaded
|
||||
"""
|
||||
global arguments
|
||||
already_downloaded = False
|
||||
|
||||
if filename and os.path.isfile(filename):
|
||||
already_downloaded = True
|
||||
if arguments['--download-archive'] and in_download_archive(track):
|
||||
already_downloaded = True
|
||||
|
||||
if already_downloaded:
|
||||
if arguments['-c'] or arguments['--remove']:
|
||||
logger.info('Track "{0}" already downloaded.'.format(title))
|
||||
return True
|
||||
else:
|
||||
logger.error('Track "{0}" already exists! Exiting... (run again with -c to continue)'.format(title))
|
||||
sys.exit(0)
|
||||
return False
|
||||
|
||||
|
||||
def in_download_archive(track):
|
||||
"""
|
||||
Returns True if a track_id exists in the download archive
|
||||
"""
|
||||
global arguments
|
||||
if not arguments['--download-archive']: return
|
||||
|
||||
archive_filename = arguments.get('--download-archive')
|
||||
try:
|
||||
with open(archive_filename, 'a+', encoding='utf-8') as file:
|
||||
logger.debug('Contents of {0}:'.format(archive_filename))
|
||||
file.seek(0)
|
||||
track_id = '{0}'.format(track['id'])
|
||||
for line in file:
|
||||
logger.debug('"'+line.strip()+'"')
|
||||
if line.strip() == track_id:
|
||||
return True
|
||||
except IOError as ioe:
|
||||
logger.error('Error trying to read download archive...')
|
||||
logger.debug(ioe)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def record_download_archive(track):
|
||||
"""
|
||||
Write the track_id in the download archive
|
||||
"""
|
||||
global arguments
|
||||
if not arguments['--download-archive']: return
|
||||
|
||||
archive_filename = arguments.get('--download-archive')
|
||||
try:
|
||||
with open(archive_filename, 'a', encoding='utf-8') as file:
|
||||
file.write('{0}'.format(track['id'])+'\n')
|
||||
except IOError as ioe:
|
||||
logger.error('Error trying to write to download archive...')
|
||||
logger.debug(ioe)
|
||||
|
||||
|
||||
def set_metadata(track, filename, album=None):
|
||||
|
|
|
|||
Loading…
Reference in New Issue