d2a242afa4
* the list of contributors is now in a separate file, `AUTHORS`. Why? Because this is meta-information which applies to the entire project, not just one particular module. Also, this file can and should be extended to indicate the authors' different roles (owner, maintainer, contributor, etc). Many successful projects follow this layout, e.g. [django](https://github.com/django/django/blob/master/AUTHORS) and [Swift](https://github.com/openstack/swift). We should adopt it as well. * Argument parsing has been extracted to `configuration.py` and the names have been changed to follow PEP-8. Unnecessarily complicated boolean conditions have been simplified * unused imports have been removed ## TESTING I did my best not to introduce any bugs in the process, but it doesn't seem we have any tests at all for the command-line interface and option parsing, so this will require thorough review! I'm looking forward to any comments and improvements, and please let me know if I broke `devscripts/bash-completion.py` or not.
333 lines
13 KiB
Python
333 lines
13 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import codecs
|
|
import io
|
|
import os
|
|
import random
|
|
import sys
|
|
|
|
from .configuration import parse_options
|
|
from .utils import (
|
|
compat_getpass,
|
|
compat_print,
|
|
DateRange,
|
|
DEFAULT_OUTTMPL,
|
|
decodeOption,
|
|
DownloadError,
|
|
MaxDownloadsReached,
|
|
preferredencoding,
|
|
read_batch_urls,
|
|
SameFileError,
|
|
setproctitle,
|
|
std_headers,
|
|
write_string,
|
|
)
|
|
from .update import update_self
|
|
from .FileDownloader import (
|
|
FileDownloader,
|
|
)
|
|
from .extractor import gen_extractors
|
|
from .YoutubeDL import YoutubeDL
|
|
from .postprocessor import (
|
|
AtomicParsleyPP,
|
|
FFmpegAudioFixPP,
|
|
FFmpegMetadataPP,
|
|
FFmpegVideoConvertor,
|
|
FFmpegExtractAudioPP,
|
|
FFmpegEmbedSubtitlePP,
|
|
XAttrMetadataPP,
|
|
)
|
|
|
|
|
|
def _real_main(argv):
|
|
# Compatibility fixes for Windows
|
|
if sys.platform == 'win32':
|
|
# https://github.com/rg3/youtube-dl/issues/820
|
|
codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
|
|
|
|
setproctitle(u'youtube-dl')
|
|
|
|
parser, opts, args = parse_options(argv)
|
|
|
|
if opts.user_agent is not None:
|
|
std_headers['User-Agent'] = opts.user_agent
|
|
|
|
if opts.referer is not None:
|
|
std_headers['Referer'] = opts.referer
|
|
|
|
if opts.headers is not None:
|
|
for h in opts.headers:
|
|
|
|
if ':' not in h[1:]:
|
|
parser.error(u'wrong header formatting, it should be key:value, not "%s"' % h)
|
|
key, value = h.split(':', 2)
|
|
if opts.verbose:
|
|
write_string(u'[debug] Adding header from command line option %s:%s\n' % (key, value))
|
|
std_headers[key] = value
|
|
|
|
if opts.dump_user_agent:
|
|
compat_print(std_headers['User-Agent'])
|
|
return
|
|
|
|
batch_urls = []
|
|
if opts.batchfile is not None:
|
|
try:
|
|
if opts.batchfile == '-':
|
|
batchfd = sys.stdin
|
|
else:
|
|
batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
|
|
batch_urls = read_batch_urls(batchfd)
|
|
if opts.verbose:
|
|
write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
|
|
except IOError:
|
|
sys.exit(u'ERROR: batch file could not be read')
|
|
|
|
all_urls = [url.strip() for url in batch_urls + args]
|
|
_enc = preferredencoding()
|
|
all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
|
|
|
|
extractors = gen_extractors()
|
|
|
|
if opts.list_extractors:
|
|
for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
|
|
status = ' (CURRENTLY BROKEN)' if not ie.working() else ''
|
|
compat_print(ie.IE_NAME + status)
|
|
for url in all_urls:
|
|
if ie.suitable(url):
|
|
compat_print(u' ' + url)
|
|
return
|
|
if opts.list_extractor_descriptions:
|
|
for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
|
|
if not ie.working():
|
|
continue
|
|
desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
|
|
if not desc:
|
|
continue
|
|
if hasattr(ie, 'SEARCH_KEY'):
|
|
searches = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
|
|
counts = (u'', u'5', u'10', u'all')
|
|
desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(counts), random.choice(searches))
|
|
compat_print(desc)
|
|
return
|
|
|
|
# Conflicting, missing and erroneous options
|
|
if opts.usenetrc and (opts.username or opts.password):
|
|
parser.error(u'using .netrc conflicts with giving username/password')
|
|
if opts.password and not opts.username:
|
|
parser.error(u'account username missing')
|
|
if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
|
|
parser.error(u'using output template conflicts with using title, video ID or auto number')
|
|
if opts.usetitle and opts.useid:
|
|
parser.error(u'using title conflicts with using video ID')
|
|
if opts.username and not opts.password:
|
|
opts.password = compat_getpass(u'Type account password and press [Return]: ')
|
|
if opts.ratelimit is not None:
|
|
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
|
|
if numeric_limit is None:
|
|
parser.error(u'invalid rate limit specified')
|
|
opts.ratelimit = numeric_limit
|
|
if opts.min_filesize is not None:
|
|
numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
|
|
if numeric_limit is None:
|
|
parser.error(u'invalid min_filesize specified')
|
|
opts.min_filesize = numeric_limit
|
|
if opts.max_filesize is not None:
|
|
numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
|
|
if numeric_limit is None:
|
|
parser.error(u'invalid max_filesize specified')
|
|
opts.max_filesize = numeric_limit
|
|
if opts.retries is not None:
|
|
try:
|
|
opts.retries = int(opts.retries)
|
|
except (TypeError, ValueError):
|
|
parser.error(u'invalid retry count specified')
|
|
if opts.buffersize is not None:
|
|
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
|
|
if numeric_buffersize is None:
|
|
parser.error(u'invalid buffer size specified')
|
|
opts.buffersize = numeric_buffersize
|
|
if opts.playliststart <= 0:
|
|
parser.error(u'Playlist start must be positive')
|
|
if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
|
|
parser.error(u'Playlist end must be greater than playlist start')
|
|
if opts.extractaudio:
|
|
if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
|
|
parser.error(u'invalid audio format specified')
|
|
if opts.audioquality:
|
|
opts.audioquality = opts.audioquality.strip('kK')
|
|
if not opts.audioquality.isdigit():
|
|
parser.error(u'invalid audio quality specified')
|
|
if opts.recodevideo:
|
|
if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
|
|
parser.error(u'invalid video recode format specified')
|
|
if opts.date is not None:
|
|
date = DateRange.day(opts.date)
|
|
else:
|
|
date = DateRange(opts.dateafter, opts.datebefore)
|
|
if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
|
|
parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
|
|
|
|
# Do not download videos when there are audio-only formats
|
|
if opts.extractaudio and not opts.keepvideo and opts.format is None:
|
|
opts.format = 'bestaudio/best'
|
|
|
|
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
|
|
# this was the old behaviour if only --all-sub was given.
|
|
if opts.allsubtitles and not opts.writeautomaticsub:
|
|
opts.writesubtitles = True
|
|
|
|
if sys.version_info < (3,):
|
|
# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
|
|
if opts.outtmpl is not None:
|
|
opts.outtmpl = opts.outtmpl.decode(preferredencoding())
|
|
outtmpl =((opts.outtmpl is not None and opts.outtmpl)
|
|
or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
|
|
or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
|
|
or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
|
|
or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
|
|
or (opts.useid and u'%(id)s.%(ext)s')
|
|
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
|
|
or DEFAULT_OUTTMPL)
|
|
if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
|
|
parser.error(u'Cannot download a video and extract audio into the same'
|
|
u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
|
|
u' template'.format(outtmpl))
|
|
|
|
any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
|
|
download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
|
|
|
|
ydl_opts = {
|
|
'usenetrc': opts.usenetrc,
|
|
'username': opts.username,
|
|
'password': opts.password,
|
|
'videopassword': opts.videopassword,
|
|
'quiet': (opts.quiet or any_printing),
|
|
'no_warnings': opts.no_warnings,
|
|
'forceurl': opts.geturl,
|
|
'forcetitle': opts.gettitle,
|
|
'forceid': opts.getid,
|
|
'forcethumbnail': opts.getthumbnail,
|
|
'forcedescription': opts.getdescription,
|
|
'forceduration': opts.getduration,
|
|
'forcefilename': opts.getfilename,
|
|
'forceformat': opts.getformat,
|
|
'forcejson': opts.dumpjson,
|
|
'simulate': opts.simulate,
|
|
'skip_download': (opts.skip_download or opts.simulate or any_printing),
|
|
'format': opts.format,
|
|
'format_limit': opts.format_limit,
|
|
'listformats': opts.listformats,
|
|
'outtmpl': outtmpl,
|
|
'autonumber_size': opts.autonumber_size,
|
|
'restrictfilenames': opts.restrictfilenames,
|
|
'ignoreerrors': opts.ignoreerrors,
|
|
'ratelimit': opts.ratelimit,
|
|
'nooverwrites': opts.nooverwrites,
|
|
'retries': opts.retries,
|
|
'buffersize': opts.buffersize,
|
|
'noresizebuffer': opts.noresizebuffer,
|
|
'continuedl': opts.continue_dl,
|
|
'noprogress': opts.noprogress,
|
|
'progress_with_newline': opts.progress_with_newline,
|
|
'playliststart': opts.playliststart,
|
|
'playlistend': opts.playlistend,
|
|
'noplaylist': opts.noplaylist,
|
|
'logtostderr': opts.outtmpl == '-',
|
|
'consoletitle': opts.consoletitle,
|
|
'nopart': opts.nopart,
|
|
'updatetime': opts.updatetime,
|
|
'writedescription': opts.writedescription,
|
|
'writeannotations': opts.writeannotations,
|
|
'writeinfojson': opts.writeinfojson,
|
|
'writethumbnail': opts.writethumbnail,
|
|
'writesubtitles': opts.writesubtitles,
|
|
'writeautomaticsub': opts.writeautomaticsub,
|
|
'allsubtitles': opts.allsubtitles,
|
|
'listsubtitles': opts.listsubtitles,
|
|
'subtitlesformat': opts.subtitlesformat,
|
|
'subtitleslangs': opts.subtitleslangs,
|
|
'matchtitle': decodeOption(opts.matchtitle),
|
|
'rejecttitle': decodeOption(opts.rejecttitle),
|
|
'max_downloads': opts.max_downloads,
|
|
'prefer_free_formats': opts.prefer_free_formats,
|
|
'verbose': opts.verbose,
|
|
'dump_intermediate_pages': opts.dump_intermediate_pages,
|
|
'write_pages': opts.write_pages,
|
|
'test': opts.test,
|
|
'keepvideo': opts.keepvideo,
|
|
'min_filesize': opts.min_filesize,
|
|
'max_filesize': opts.max_filesize,
|
|
'min_views': opts.min_views,
|
|
'max_views': opts.max_views,
|
|
'daterange': date,
|
|
'cachedir': opts.cachedir,
|
|
'youtube_print_sig_code': opts.youtube_print_sig_code,
|
|
'age_limit': opts.age_limit,
|
|
'download_archive': download_archive_fn,
|
|
'cookiefile': opts.cookiefile,
|
|
'nocheckcertificate': opts.no_check_certificate,
|
|
'prefer_insecure': opts.prefer_insecure,
|
|
'proxy': opts.proxy,
|
|
'socket_timeout': opts.socket_timeout,
|
|
'bidi_workaround': opts.bidi_workaround,
|
|
'debug_printtraffic': opts.debug_printtraffic,
|
|
'prefer_ffmpeg': opts.prefer_ffmpeg,
|
|
'include_ads': opts.include_ads,
|
|
'default_search': opts.default_search,
|
|
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
|
|
'encoding': opts.encoding,
|
|
}
|
|
|
|
with YoutubeDL(ydl_opts) as ydl:
|
|
ydl.print_debug_header()
|
|
ydl.add_default_info_extractors()
|
|
|
|
# PostProcessors
|
|
# Add the metadata pp first, the other pps will copy it
|
|
if opts.addmetadata:
|
|
ydl.add_post_processor(FFmpegMetadataPP())
|
|
if opts.extractaudio:
|
|
ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
|
|
if opts.recodevideo:
|
|
ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
|
|
if opts.embedsubtitles:
|
|
ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
|
|
if opts.xattrs:
|
|
ydl.add_post_processor(XAttrMetadataPP())
|
|
if opts.embedthumbnail:
|
|
if not opts.addmetadata:
|
|
ydl.add_post_processor(FFmpegAudioFixPP())
|
|
ydl.add_post_processor(AtomicParsleyPP())
|
|
|
|
# Update version
|
|
if opts.update_self:
|
|
update_self(ydl.to_screen, opts.verbose)
|
|
return
|
|
|
|
# Maybe do nothing
|
|
if not (all_urls or opts.load_info_filename):
|
|
parser.error(u'you must provide at least one URL')
|
|
|
|
try:
|
|
if opts.load_info_filename:
|
|
retcode = ydl.download_with_info_file(opts.load_info_filename)
|
|
else:
|
|
retcode = ydl.download(all_urls)
|
|
except MaxDownloadsReached:
|
|
ydl.to_screen(u'--max-download limit reached, aborting.')
|
|
retcode = 101
|
|
|
|
sys.exit(retcode)
|
|
|
|
|
|
def main(argv=None):
|
|
try:
|
|
_real_main(argv)
|
|
except DownloadError:
|
|
sys.exit(1)
|
|
except SameFileError:
|
|
sys.exit(u'ERROR: fixed output name but more than one file to download')
|
|
except KeyboardInterrupt:
|
|
sys.exit(u'\nERROR: Interrupted by user')
|