R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

YoutubeDL: use parameters of the specific extractor

Browse Source 
The 'force*' and 'extract_flat' parameters are only looked in the main parameters dict, because I think that it only makes sense to use them in a global way
This commit is contained in:
Jaime Marquínez Ferrándiz 2016-03-18 22:30:21 +01:00
parent b67e4daddb
commit 09495f1d12
3 changed files with 87 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
test/test_YoutubeDL.py
View File

@ -28,7 +28,7 @@ class YDL(FakeYDL):
self.downloaded_info_dicts = []
self.msgs = []
def process_info(self, info_dict):
def process_info(self, info_dict, params):
self.downloaded_info_dicts.append(info_dict)
def to_screen(self, msg):
@ -438,7 +438,7 @@ class TestYoutubeDL(unittest.TestCase):
params.setdefault('simulate', True)
ydl = YDL(params)
ydl.report_warning = lambda *args, **kargs: None
return ydl.process_video_result(info_dict, download=False)
return ydl.process_video_result(info_dict, params, download=False)
result = get_info()
self.assertFalse(result.get('requested_subtitles'))
@ -527,7 +527,7 @@ class TestYoutubeDL(unittest.TestCase):
f.write('EXAMPLE')
ydl = YoutubeDL(params)
ydl.add_post_processor(PP())
ydl.post_process(filename, {'filepath': filename})
ydl.post_process(filename, {'filepath': filename}, params)
run_pp({'keepvideo': True}, SimplePP)
self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
@ -556,8 +556,8 @@ class TestYoutubeDL(unittest.TestCase):
super(FilterYDL, self).__init__(*args, **kwargs)
self.params['simulate'] = True
def process_info(self, info_dict):
super(YDL, self).process_info(info_dict)
def process_info(self, info_dict, params):
super(YDL, self).process_info(info_dict, params)
def _match_entry(self, info_dict, incomplete):
res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
@ -704,12 +704,30 @@ class TestYoutubeDL(unittest.TestCase):
self.assertEqual(downloaded['url'], TEST_URL)
def test_subparams(self):
example_params = {'foo': 'example'}
params = Params({'foo': 'base', 'blah': 'base'}, {'example.com': example_params})
ydl = YoutubeDL(params)
example_params = {'foo': 'example', 'outtmpl': 'foo.mp4'}
params = Params(
{'foo': 'base', 'blah': 'base', 'skip_download': True}, {'example.com': example_params})
ydl = YoutubeDL(params, auto_init=False)
ydl.downloads = []
real_process_info = ydl.process_info
def process_info(info_dict, params):
r = real_process_info(info_dict, params)
ydl.downloads.append(info_dict)
return r
ydl.process_info = process_info
class ExampleIE(InfoExtractor):
IE_NAME = 'example.com'
_VALID_URL = r'example$'
def _real_extract(self, url):
return {
'id': '1',
'ext': 'mp4',
'title': 'example',
'url': 'http://example.com',
}
ie = ExampleIE()
ydl.add_info_extractor(ie)
@ -719,6 +737,9 @@ class TestYoutubeDL(unittest.TestCase):
self.assertEqual(pars.get('blah'), 'base')
self.assertEqual(pars.get('nonexistant'), None)
ydl.extract_info('example')
self.assertEqual(ydl.downloads[-1]['_filename'], 'foo.mp4')
if __name__ == '__main__':
unittest.main()

4
test/test_download.py
View File

@ -51,9 +51,9 @@ class YoutubeDL(youtube_dl.YoutubeDL):
# Don't accept warnings during tests
raise ExtractorError(message)
def process_info(self, info_dict):
def process_info(self, info_dict, params):
self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict)
return super(YoutubeDL, self).process_info(info_dict, params)
def _file_md5(fn):

101
youtube_dl/YoutubeDL.py
View File

@ -556,13 +556,15 @@ class YoutubeDL(object):
except UnicodeEncodeError:
self.to_screen('[download] The file has already been downloaded')
def prepare_filename(self, info_dict):
def prepare_filename(self, info_dict, params=None):
"""Generate the output filename."""
if params is None:
params = self.params
try:
template_dict = dict(info_dict)
template_dict['epoch'] = int(time.time())
autonumber_size = self.params.get('autonumber_size')
autonumber_size = params.get('autonumber_size')
if autonumber_size is None:
autonumber_size = 5
autonumber_templ = '%0' + str(autonumber_size) + 'd'
@ -579,14 +581,14 @@ class YoutubeDL(object):
sanitize = lambda k, v: sanitize_filename(
compat_str(v),
restricted=self.params.get('restrictfilenames'),
restricted=params.get('restrictfilenames'),
is_id=(k == 'id'))
template_dict = dict((k, sanitize(k, v))
for k, v in template_dict.items()
if v is not None)
template_dict = collections.defaultdict(lambda: 'NA', template_dict)
outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
outtmpl = params.get('outtmpl', DEFAULT_OUTTMPL)
tmpl = compat_expanduser(outtmpl)
filename = tmpl % template_dict
# Temporary fix for #4787
@ -683,7 +685,9 @@ class YoutubeDL(object):
}
self.add_default_extra_info(ie_result, ie, url)
if process:
return self.process_ie_result(ie_result, download, extra_info)
return self.process_ie_result(
ie_result, download, extra_info,
params=self.params.section(ie.IE_NAME))
else:
return ie_result
except ExtractorError as e: # An error we somewhat expected
@ -708,7 +712,7 @@ class YoutubeDL(object):
'extractor_key': ie.ie_key(),
})
def process_ie_result(self, ie_result, download=True, extra_info={}):
def process_ie_result(self, ie_result, download=True, extra_info={}, params=None):
"""
Take the result of the ie(may be modified) and resolve all unresolved
references (URLs, playlist items).
@ -716,6 +720,8 @@ class YoutubeDL(object):
It will also download the videos if 'download'.
Returns the resolved ie_result.
"""
if params is None:
params = self.params
result_type = ie_result.get('_type', 'video')
if result_type in ('url', 'url_transparent'):
@ -728,7 +734,7 @@ class YoutubeDL(object):
if result_type == 'video':
self.add_extra_info(ie_result, extra_info)
return self.process_video_result(ie_result, download=download)
return self.process_video_result(ie_result, params, download=download)
elif result_type == 'url':
# We have to add extra_info to the results because it may be
# contained in a playlist
@ -753,7 +759,7 @@ class YoutubeDL(object):
assert new_result.get('_type') != 'url_transparent'
return self.process_ie_result(
new_result, download=download, extra_info=extra_info)
new_result, download=download, extra_info=extra_info, params=params)
elif result_type == 'playlist' or result_type == 'multi_video':
# We process each entry in the playlist
playlist = ie_result.get('title') or ie_result.get('id')
@ -843,7 +849,8 @@ class YoutubeDL(object):
entry_result = self.process_ie_result(entry,
download=download,
extra_info=extra)
extra_info=extra,
params=params)
playlist_results.append(entry_result)
ie_result['entries'] = playlist_results
self.to_screen('[download] Finished downloading playlist: %s' % playlist)
@ -865,7 +872,7 @@ class YoutubeDL(object):
)
return r
ie_result['entries'] = [
self.process_ie_result(_fixup(r), download, extra_info)
self.process_ie_result(_fixup(r), download, extra_info, params=params)
for r in ie_result['entries']
]
return ie_result
@ -1213,7 +1220,7 @@ class YoutubeDL(object):
self.cookiejar.add_cookie_header(pr)
return pr.get_header('Cookie')
def process_video_result(self, info_dict, download=True):
def process_video_result(self, info_dict, params, download=True):
assert info_dict.get('_type', 'video') == 'video'
if 'id' not in info_dict:
@ -1283,7 +1290,7 @@ class YoutubeDL(object):
return
info_dict['requested_subtitles'] = self.process_subtitles(
info_dict['id'], subtitles,
info_dict.get('automatic_captions'))
info_dict.get('automatic_captions'), params)
# We now pick which formats have to be downloaded
if info_dict.get('formats') is None:
@ -1352,10 +1359,10 @@ class YoutubeDL(object):
self.list_formats(info_dict)
return
req_format = self.params.get('format')
req_format = params.get('format')
if req_format is None:
req_format_list = []
if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
if (params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
not info_dict.get('is_live')):
merger = FFmpegMergerPP(self)
if merger.available and merger.can_merge():
@ -1374,37 +1381,37 @@ class YoutubeDL(object):
for format in formats_to_download:
new_info = dict(info_dict)
new_info.update(format)
self.process_info(new_info)
self.process_info(new_info, params)
# We update the info dict with the best quality format (backwards compatibility)
info_dict.update(formats_to_download[-1])
return info_dict
def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
def process_subtitles(self, video_id, normal_subtitles, automatic_captions, params):
"""Select the requested subtitles and their format"""
available_subs = {}
if normal_subtitles and self.params.get('writesubtitles'):
if normal_subtitles and params.get('writesubtitles'):
available_subs.update(normal_subtitles)
if automatic_captions and self.params.get('writeautomaticsub'):
if automatic_captions and params.get('writeautomaticsub'):
for lang, cap_info in automatic_captions.items():
if lang not in available_subs:
available_subs[lang] = cap_info
if (not self.params.get('writesubtitles') and not
self.params.get('writeautomaticsub') or not
if (not params.get('writesubtitles') and not
params.get('writeautomaticsub') or not
available_subs):
return None
if self.params.get('allsubtitles', False):
if params.get('allsubtitles', False):
requested_langs = available_subs.keys()
else:
if self.params.get('subtitleslangs', False):
requested_langs = self.params.get('subtitleslangs')
if params.get('subtitleslangs', False):
requested_langs = params.get('subtitleslangs')
elif 'en' in available_subs:
requested_langs = ['en']
else:
requested_langs = [list(available_subs.keys())[0]]
formats_query = self.params.get('subtitlesformat', 'best')
formats_query = params.get('subtitlesformat', 'best')
formats_preference = formats_query.split('/') if formats_query else []
subs = {}
for lang in requested_langs:
@ -1428,7 +1435,7 @@ class YoutubeDL(object):
subs[lang] = f
return subs
def process_info(self, info_dict):
def process_info(self, info_dict, params):
"""Process a single resolved IE result."""
assert info_dict.get('_type', 'video') == 'video'
@ -1452,7 +1459,7 @@ class YoutubeDL(object):
self._num_downloads += 1
info_dict['_filename'] = filename = self.prepare_filename(info_dict)
info_dict['_filename'] = filename = self.prepare_filename(info_dict, params)
# Forced printings
if self.params.get('forcetitle', False):
@ -1494,9 +1501,9 @@ class YoutubeDL(object):
self.report_error('unable to create directory ' + error_to_compat_str(err))
return
if self.params.get('writedescription', False):
if params.get('writedescription', False):
descfn = replace_extension(filename, 'description', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
if params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
self.to_screen('[info] Video description is already present')
elif info_dict.get('description') is None:
self.report_warning('There\'s no description to write.')
@ -1509,9 +1516,9 @@ class YoutubeDL(object):
self.report_error('Cannot write description file ' + descfn)
return
if self.params.get('writeannotations', False):
if params.get('writeannotations', False):
annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
if params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
self.to_screen('[info] Video annotations are already present')
else:
try:
@ -1524,8 +1531,8 @@ class YoutubeDL(object):
self.report_error('Cannot write annotations file: ' + annofn)
return
subtitles_are_requested = any([self.params.get('writesubtitles', False),
self.params.get('writeautomaticsub')])
subtitles_are_requested = any([params.get('writesubtitles', False),
params.get('writeautomaticsub')])
if subtitles_are_requested and info_dict.get('requested_subtitles'):
# subtitles download errors are already managed as troubles in relevant IE
@ -1546,7 +1553,7 @@ class YoutubeDL(object):
continue
try:
sub_filename = subtitles_filename(filename, sub_lang, sub_format)
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
if params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
else:
self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@ -1556,9 +1563,9 @@ class YoutubeDL(object):
self.report_error('Cannot write subtitles file ' + sub_filename)
return
if self.params.get('writeinfojson', False):
if params.get('writeinfojson', False):
infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
if params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
self.to_screen('[info] Video description metadata is already present')
else:
self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
@ -1570,10 +1577,10 @@ class YoutubeDL(object):
self._write_thumbnails(info_dict, filename)
if not self.params.get('skip_download', False):
if not params.get('skip_download', False):
try:
def dl(name, info):
fd = get_suitable_downloader(info, self.params)(self, self.params)
fd = get_suitable_downloader(info, params)(self, params)
for ph in self._progress_hooks:
fd.add_progress_hook(ph)
if self.params.get('verbose'):
@ -1613,7 +1620,7 @@ class YoutubeDL(object):
if filename_real_ext == info_dict['ext']
else filename)
requested_formats = info_dict['requested_formats']
if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
if params.get('merge_output_format') is None and not compatible_formats(requested_formats):
info_dict['ext'] = 'mkv'
self.report_warning(
'Requested formats are incompatible for merge and will be merged into mkv.')
@ -1627,7 +1634,7 @@ class YoutubeDL(object):
for f in requested_formats:
new_info = dict(info_dict)
new_info.update(f)
fname = self.prepare_filename(new_info)
fname = self.prepare_filename(new_info, params)
fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
downloaded.append(fname)
partial_success = dl(fname, new_info)
@ -1648,7 +1655,7 @@ class YoutubeDL(object):
if success and filename != '-':
# Fixup content
fixup_policy = self.params.get('fixup')
fixup_policy = params.get('fixup')
if fixup_policy is None:
fixup_policy = 'detect_or_warn'
@ -1693,7 +1700,7 @@ class YoutubeDL(object):
if (info_dict.get('protocol') == 'm3u8_native' or
info_dict.get('protocol') == 'm3u8' and
self.params.get('hls_prefer_native')):
params.get('hls_prefer_native')):
if fixup_policy == 'warn':
self.report_warning('%s: malformated aac bitstream.' % (
info_dict['id']))
@ -1710,7 +1717,7 @@ class YoutubeDL(object):
assert fixup_policy in ('ignore', 'never')
try:
self.post_process(filename, info_dict)
self.post_process(filename, info_dict, params)
except (PostProcessingError) as err:
self.report_error('postprocessing: %s' % str(err))
return
@ -1747,7 +1754,11 @@ class YoutubeDL(object):
# FileInput doesn't have a read method, we can't call json.load
info = self.filter_requested_info(json.loads('\n'.join(f)))
try:
self.process_ie_result(info, download=True)
params = None
ie_name = info.get('extractor')
if ie_name:
params = self.params.section(ie_name)
self.process_ie_result(info, download=True, params=params)
except DownloadError:
webpage_url = info.get('webpage_url')
if webpage_url is not None:
@ -1763,7 +1774,7 @@ class YoutubeDL(object):
(k, v) for k, v in info_dict.items()
if k not in ['requested_formats', 'requested_subtitles'])
def post_process(self, filename, ie_info):
def post_process(self, filename, ie_info, params):
"""Run all the postprocessors on the given file."""
info = dict(ie_info)
info['filepath'] = filename
@ -1777,7 +1788,7 @@ class YoutubeDL(object):
files_to_delete, info = pp.run(info)
except PostProcessingError as e:
self.report_error(e.msg)
if files_to_delete and not self.params.get('keepvideo', False):
if files_to_delete and not params.get('keepvideo', False):
for old_filename in files_to_delete:
self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
try: