l1ving_youtube-dl/youtube_dl/__init__.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs
import io
import os
import random
import sys

from .configuration import parse_options
from .utils import (
    compat_getpass,
    compat_print,
    DateRange,
    DEFAULT_OUTTMPL,
    decodeOption,
    DownloadError,
    MaxDownloadsReached,
    preferredencoding,
    read_batch_urls,
    SameFileError,
    setproctitle,
    std_headers,
    write_string,
)
from .update import update_self
from .FileDownloader import (
    FileDownloader,
)
from .extractor import gen_extractors
from .YoutubeDL import YoutubeDL
from .postprocessor import (
    AtomicParsleyPP,
    FFmpegAudioFixPP,
    FFmpegMetadataPP,
    FFmpegVideoConvertor,
    FFmpegExtractAudioPP,
    FFmpegEmbedSubtitlePP,
    XAttrMetadataPP,
)


def _real_main(argv):
    # Compatibility fixes for Windows
    if sys.platform == 'win32':
        # https://github.com/rg3/youtube-dl/issues/820
        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)

    setproctitle(u'youtube-dl')

    parser, opts, args = parse_options(argv)

    if opts.user_agent is not None:
        std_headers['User-Agent'] = opts.user_agent

    if opts.referer is not None:
        std_headers['Referer'] = opts.referer

    if opts.headers is not None:
        for h in opts.headers:

            if ':' not in h[1:]:
                parser.error(u'wrong header formatting, it should be key:value, not "%s"' % h)
            key, value = h.split(':', 2)
            if opts.verbose:
                write_string(u'[debug] Adding header from command line option %s:%s\n' % (key, value))
            std_headers[key] = value

    if opts.dump_user_agent:
        compat_print(std_headers['User-Agent'])
        return

    batch_urls = []
    if opts.batchfile is not None:
        try:
            if opts.batchfile == '-':
                batchfd = sys.stdin
            else:
                batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
            batch_urls = read_batch_urls(batchfd)
            if opts.verbose:
                write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
        except IOError:
            sys.exit(u'ERROR: batch file could not be read')

    all_urls = [url.strip() for url in batch_urls + args]
    _enc = preferredencoding()
    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]

    extractors = gen_extractors()

    if opts.list_extractors:
        for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
            status = ' (CURRENTLY BROKEN)' if not ie.working() else ''
            compat_print(ie.IE_NAME + status)
            for url in all_urls:
                if ie.suitable(url):
                    compat_print(u'  ' + url)
        return
    if opts.list_extractor_descriptions:
        for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
            if not ie.working():
                continue
            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
            if not desc:
                continue
            if hasattr(ie, 'SEARCH_KEY'):
                searches = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
                counts = (u'', u'5', u'10', u'all')
                desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(counts), random.choice(searches))
            compat_print(desc)
        return

    # Conflicting, missing and erroneous options
    if opts.usenetrc and (opts.username or opts.password):
        parser.error(u'using .netrc conflicts with giving username/password')
    if opts.password and not opts.username:
        parser.error(u'account username missing')
    if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
        parser.error(u'using output template conflicts with using title, video ID or auto number')
    if opts.usetitle and opts.useid:
        parser.error(u'using title conflicts with using video ID')
    if opts.username and not opts.password:
        opts.password = compat_getpass(u'Type account password and press [Return]: ')
    if opts.ratelimit is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
        if numeric_limit is None:
            parser.error(u'invalid rate limit specified')
        opts.ratelimit = numeric_limit
    if opts.min_filesize is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
        if numeric_limit is None:
            parser.error(u'invalid min_filesize specified')
        opts.min_filesize = numeric_limit
    if opts.max_filesize is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
        if numeric_limit is None:
            parser.error(u'invalid max_filesize specified')
        opts.max_filesize = numeric_limit
    if opts.retries is not None:
        try:
            opts.retries = int(opts.retries)
        except (TypeError, ValueError):
            parser.error(u'invalid retry count specified')
    if opts.buffersize is not None:
        numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
        if numeric_buffersize is None:
            parser.error(u'invalid buffer size specified')
        opts.buffersize = numeric_buffersize
    if opts.playliststart <= 0:
        parser.error(u'Playlist start must be positive')
    if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
        parser.error(u'Playlist end must be greater than playlist start')
    if opts.extractaudio:
        if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
            parser.error(u'invalid audio format specified')
    if opts.audioquality:
        opts.audioquality = opts.audioquality.strip('kK')
        if not opts.audioquality.isdigit():
            parser.error(u'invalid audio quality specified')
    if opts.recodevideo:
        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
            parser.error(u'invalid video recode format specified')
    if opts.date is not None:
        date = DateRange.day(opts.date)
    else:
        date = DateRange(opts.dateafter, opts.datebefore)
    if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
        parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')

    # Do not download videos when there are audio-only formats
    if opts.extractaudio and not opts.keepvideo and opts.format is None:
        opts.format = 'bestaudio/best'

    # --all-sub automatically sets --write-sub if --write-auto-sub is not given
    # this was the old behaviour if only --all-sub was given.
    if opts.allsubtitles and not opts.writeautomaticsub:
        opts.writesubtitles = True

    if sys.version_info < (3,):
        # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
        if opts.outtmpl is not None:
            opts.outtmpl = opts.outtmpl.decode(preferredencoding())
    outtmpl =((opts.outtmpl is not None and opts.outtmpl)
            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
            or (opts.useid and u'%(id)s.%(ext)s')
            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
            or DEFAULT_OUTTMPL)
    if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
        parser.error(u'Cannot download a video and extract audio into the same'
                     u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                     u' template'.format(outtmpl))

    any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
    download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive

    ydl_opts = {
        'usenetrc': opts.usenetrc,
        'username': opts.username,
        'password': opts.password,
        'videopassword': opts.videopassword,
        'quiet': (opts.quiet or any_printing),
        'no_warnings': opts.no_warnings,
        'forceurl': opts.geturl,
        'forcetitle': opts.gettitle,
        'forceid': opts.getid,
        'forcethumbnail': opts.getthumbnail,
        'forcedescription': opts.getdescription,
        'forceduration': opts.getduration,
        'forcefilename': opts.getfilename,
        'forceformat': opts.getformat,
        'forcejson': opts.dumpjson,
        'simulate': opts.simulate,
        'skip_download': (opts.skip_download or opts.simulate or any_printing),
        'format': opts.format,
        'format_limit': opts.format_limit,
        'listformats': opts.listformats,
        'outtmpl': outtmpl,
        'autonumber_size': opts.autonumber_size,
        'restrictfilenames': opts.restrictfilenames,
        'ignoreerrors': opts.ignoreerrors,
        'ratelimit': opts.ratelimit,
        'nooverwrites': opts.nooverwrites,
        'retries': opts.retries,
        'buffersize': opts.buffersize,
        'noresizebuffer': opts.noresizebuffer,
        'continuedl': opts.continue_dl,
        'noprogress': opts.noprogress,
        'progress_with_newline': opts.progress_with_newline,
        'playliststart': opts.playliststart,
        'playlistend': opts.playlistend,
        'noplaylist': opts.noplaylist,
        'logtostderr': opts.outtmpl == '-',
        'consoletitle': opts.consoletitle,
        'nopart': opts.nopart,
        'updatetime': opts.updatetime,
        'writedescription': opts.writedescription,
        'writeannotations': opts.writeannotations,
        'writeinfojson': opts.writeinfojson,
        'writethumbnail': opts.writethumbnail,
        'writesubtitles': opts.writesubtitles,
        'writeautomaticsub': opts.writeautomaticsub,
        'allsubtitles': opts.allsubtitles,
        'listsubtitles': opts.listsubtitles,
        'subtitlesformat': opts.subtitlesformat,
        'subtitleslangs': opts.subtitleslangs,
        'matchtitle': decodeOption(opts.matchtitle),
        'rejecttitle': decodeOption(opts.rejecttitle),
        'max_downloads': opts.max_downloads,
        'prefer_free_formats': opts.prefer_free_formats,
        'verbose': opts.verbose,
        'dump_intermediate_pages': opts.dump_intermediate_pages,
        'write_pages': opts.write_pages,
        'test': opts.test,
        'keepvideo': opts.keepvideo,
        'min_filesize': opts.min_filesize,
        'max_filesize': opts.max_filesize,
        'min_views': opts.min_views,
        'max_views': opts.max_views,
        'daterange': date,
        'cachedir': opts.cachedir,
        'youtube_print_sig_code': opts.youtube_print_sig_code,
        'age_limit': opts.age_limit,
        'download_archive': download_archive_fn,
        'cookiefile': opts.cookiefile,
        'nocheckcertificate': opts.no_check_certificate,
        'prefer_insecure': opts.prefer_insecure,
        'proxy': opts.proxy,
        'socket_timeout': opts.socket_timeout,
        'bidi_workaround': opts.bidi_workaround,
        'debug_printtraffic': opts.debug_printtraffic,
        'prefer_ffmpeg': opts.prefer_ffmpeg,
        'include_ads': opts.include_ads,
        'default_search': opts.default_search,
        'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
        'encoding': opts.encoding,
    }

    with YoutubeDL(ydl_opts) as ydl:
        ydl.print_debug_header()
        ydl.add_default_info_extractors()

        # PostProcessors
        # Add the metadata pp first, the other pps will copy it
        if opts.addmetadata:
            ydl.add_post_processor(FFmpegMetadataPP())
        if opts.extractaudio:
            ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
        if opts.recodevideo:
            ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
        if opts.embedsubtitles:
            ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
        if opts.xattrs:
            ydl.add_post_processor(XAttrMetadataPP())
        if opts.embedthumbnail:
            if not opts.addmetadata:
                ydl.add_post_processor(FFmpegAudioFixPP())
            ydl.add_post_processor(AtomicParsleyPP())

        # Update version
        if opts.update_self:
            update_self(ydl.to_screen, opts.verbose)
            return

        # Maybe do nothing
        if not (all_urls or opts.load_info_filename):
            parser.error(u'you must provide at least one URL')

        try:
            if opts.load_info_filename:
                retcode = ydl.download_with_info_file(opts.load_info_filename)
            else:
                retcode = ydl.download(all_urls)
        except MaxDownloadsReached:
            ydl.to_screen(u'--max-download limit reached, aborting.')
            retcode = 101

    sys.exit(retcode)


def main(argv=None):
    try:
        _real_main(argv)
    except DownloadError:
        sys.exit(1)
    except SameFileError:
        sys.exit(u'ERROR: fixed output name but more than one file to download')
    except KeyboardInterrupt:
        sys.exit(u'\nERROR: Interrupted by user')