diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 55b22c889..1842e9e22 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -372,6 +372,7 @@ def _real_main(argv=None): 'external_downloader_args': external_downloader_args, 'postprocessor_args': postprocessor_args, 'cn_verification_proxy': opts.cn_verification_proxy, + 'parallel_dash_downloads': opts.parallel_dash_downloads, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index ad971772b..716e0cfa7 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -14,6 +14,10 @@ from ..utils import ( ) +class StopDownload(Exception): + pass + + class FileDownloader(object): """File Downloader class. @@ -232,6 +236,9 @@ class FileDownloader(object): self.to_console_title('youtube-dl ' + msg) def report_progress(self, s): + if s.get('_skip_report_progress'): + return + if s['status'] == 'finished': if self.params.get('noprogress', False): self.to_screen('[download] Download completed') diff --git a/youtube_dl/downloader/merge.py b/youtube_dl/downloader/merge.py index 7eadeb9b1..5774dfd07 100644 --- a/youtube_dl/downloader/merge.py +++ b/youtube_dl/downloader/merge.py @@ -1,20 +1,19 @@ -from __future__ import unicode_literals +from __future__ import unicode_literals, division -from .common import FileDownloader +import threading + +from .common import FileDownloader, StopDownload import youtube_dl from ..utils import prepend_extension +def _join_threads(threads, timeout=None): + for t in threads: + t.join(timeout=timeout) + + class MergeFD(FileDownloader): - def real_download(self, filename, info_dict): - infos = [] - for f in info_dict['requested_formats']: - new_info = dict(info_dict) - del new_info['requested_formats'] - new_info.update(f) - fname = self.ydl.prepare_filename(new_info) - fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext']) - infos.append((fname, new_info)) + def _normal_download(self, filename, infos): success = True for fname, info in infos: params = dict(self.params) @@ -31,7 +30,97 @@ class MergeFD(FileDownloader): self.report_destination(fname) partial_success = fd.download(fname, info) success = success and partial_success + return success - info_dict['__files_to_merge'] = [fname for fname, _ in infos] + def _parallel_download(self, filename, infos): + self.report_warning('Downloading DASH formats in parallel is an experimental feature, some things may not work as expected') + threads = [] + statuses = [] + downloaders = [] + lock = threading.Lock() + stop_event = threading.Event() + for fname, info in infos: + params = dict(self.params) + params.update({ + 'quiet': True, + 'noprogress': True, + }) + FD = youtube_dl.downloader.get_suitable_downloader(info, self.params) + fd = FD(self.ydl, params) + downloaders.append(fd) + status = {} + statuses.append(status) + + def hook(s, status=status): + with lock: + status.update(s) + s['_skip_report_progress'] = True + self._hook_progress(s) + + global_status = {'filename': filename} + if any(s.get('status') == 'downloading' for s in statuses): + global_status['status'] = 'downloading' + elif all(s.get('status') == 'finished' for s in statuses): + global_status['status'] = 'finished' + else: + global_status['status'] = None + for s in statuses: + for key in ['total_bytes', 'downloaded_bytes', 'eta', 'elapsed', 'speed']: + if s.get(key) is not None: + global_status.setdefault(key, 0) + global_status[key] += s[key] + # Don't call _hook_progress because it's not a real file + self.report_progress(global_status) + if stop_event.is_set(): + raise StopDownload() + + fd.add_progress_hook(hook) + self.report_destination(fname) + + def dl(fd, *args): + fd._error = None + try: + return fd.download(*args) + except StopDownload: + pass + except Exception as err: + fd._error = err + + thread = threading.Thread(target=dl, args=(fd, fname, info)) + threads.append(thread) + try: + for t in threads: + t.start() + while True: + # the timeout seems to be required so that the main thread can + # catch the exceptions in python 2.x + _join_threads(threads, timeout=1) + if not any(t.is_alive() for t in threads): + break + except BaseException: + stop_event.set() + _join_threads(threads) + raise + + for fd in downloaders: + if fd._error is not None: + raise fd._error return True + + def real_download(self, filename, info_dict): + infos = [] + for f in info_dict['requested_formats']: + new_info = dict(info_dict) + del new_info['requested_formats'] + new_info.update(f) + fname = self.ydl.prepare_filename(new_info) + fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext']) + infos.append((fname, new_info)) + + info_dict['__files_to_merge'] = [name for name, _ in infos] + + if self.params.get('parallel_dash_downloads', False): + return self._parallel_download(filename, infos) + else: + return self._normal_download(filename, infos) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 85365d769..eb199dfb2 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -424,6 +424,10 @@ def parseOpts(overrideArguments=None): '--external-downloader-args', dest='external_downloader_args', metavar='ARGS', help='Give these arguments to the external downloader') + downloader.add_option( + '--parallel-dash-downloads', + action='store_true', dest='parallel_dash_downloads', default=False, + help='(Experimental) download dash formats in parallel') workarounds = optparse.OptionGroup(parser, 'Workarounds') workarounds.add_option(