R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

YoutubeDL: use parameters of the specific extractor

Browse Source 
The 'force*' and 'extract_flat' parameters are only looked in the main parameters dict, because I think that it only makes sense to use them in a global way
This commit is contained in:
Jaime Marquínez Ferrándiz 2016-03-18 22:30:21 +01:00
parent b67e4daddb
commit 09495f1d12
3 changed files with 87 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
test/test_YoutubeDL.py
View File

@ -28,7 +28,7 @@ class YDL(FakeYDL):
self.downloaded_info_dicts = [] self.downloaded_info_dicts = []
self.msgs = [] self.msgs = []
def process_info(self, info_dict): def process_info(self, info_dict, params):
self.downloaded_info_dicts.append(info_dict) self.downloaded_info_dicts.append(info_dict)
def to_screen(self, msg): def to_screen(self, msg):
@ -438,7 +438,7 @@ class TestYoutubeDL(unittest.TestCase):
params.setdefault('simulate', True) params.setdefault('simulate', True)
ydl = YDL(params) ydl = YDL(params)
ydl.report_warning = lambda *args, **kargs: None ydl.report_warning = lambda *args, **kargs: None
return ydl.process_video_result(info_dict, download=False) return ydl.process_video_result(info_dict, params, download=False)
result = get_info() result = get_info()
self.assertFalse(result.get('requested_subtitles')) self.assertFalse(result.get('requested_subtitles'))
@ -527,7 +527,7 @@ class TestYoutubeDL(unittest.TestCase):
f.write('EXAMPLE') f.write('EXAMPLE')
ydl = YoutubeDL(params) ydl = YoutubeDL(params)
ydl.add_post_processor(PP()) ydl.add_post_processor(PP())
ydl.post_process(filename, {'filepath': filename}) ydl.post_process(filename, {'filepath': filename}, params)
run_pp({'keepvideo': True}, SimplePP) run_pp({'keepvideo': True}, SimplePP)
self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename) self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
@ -556,8 +556,8 @@ class TestYoutubeDL(unittest.TestCase):
super(FilterYDL, self).__init__(*args, **kwargs) super(FilterYDL, self).__init__(*args, **kwargs)
self.params['simulate'] = True self.params['simulate'] = True
def process_info(self, info_dict): def process_info(self, info_dict, params):
super(YDL, self).process_info(info_dict) super(YDL, self).process_info(info_dict, params)
def _match_entry(self, info_dict, incomplete): def _match_entry(self, info_dict, incomplete):
res = super(FilterYDL, self)._match_entry(info_dict, incomplete) res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
@ -704,12 +704,30 @@ class TestYoutubeDL(unittest.TestCase):
self.assertEqual(downloaded['url'], TEST_URL) self.assertEqual(downloaded['url'], TEST_URL)
def test_subparams(self): def test_subparams(self):
example_params = {'foo': 'example'} example_params = {'foo': 'example', 'outtmpl': 'foo.mp4'}
params = Params({'foo': 'base', 'blah': 'base'}, {'example.com': example_params}) params = Params(
ydl = YoutubeDL(params) {'foo': 'base', 'blah': 'base', 'skip_download': True}, {'example.com': example_params})
ydl = YoutubeDL(params, auto_init=False)
ydl.downloads = []
real_process_info = ydl.process_info
def process_info(info_dict, params):
r = real_process_info(info_dict, params)
ydl.downloads.append(info_dict)
return r
ydl.process_info = process_info
class ExampleIE(InfoExtractor): class ExampleIE(InfoExtractor):
IE_NAME = 'example.com' IE_NAME = 'example.com'
_VALID_URL = r'example$'
def _real_extract(self, url):
return {
'id': '1',
'ext': 'mp4',
'title': 'example',
'url': 'http://example.com',
}
ie = ExampleIE() ie = ExampleIE()
ydl.add_info_extractor(ie) ydl.add_info_extractor(ie)
@ -719,6 +737,9 @@ class TestYoutubeDL(unittest.TestCase):
self.assertEqual(pars.get('blah'), 'base') self.assertEqual(pars.get('blah'), 'base')
self.assertEqual(pars.get('nonexistant'), None) self.assertEqual(pars.get('nonexistant'), None)
ydl.extract_info('example')
self.assertEqual(ydl.downloads[-1]['_filename'], 'foo.mp4')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

4
test/test_download.py
View File

@ -51,9 +51,9 @@ class YoutubeDL(youtube_dl.YoutubeDL):
# Don't accept warnings during tests # Don't accept warnings during tests
raise ExtractorError(message) raise ExtractorError(message)
def process_info(self, info_dict): def process_info(self, info_dict, params):
self.processed_info_dicts.append(info_dict) self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict) return super(YoutubeDL, self).process_info(info_dict, params)
def _file_md5(fn): def _file_md5(fn):

101
youtube_dl/YoutubeDL.py
View File

@ -556,13 +556,15 @@ class YoutubeDL(object):
except UnicodeEncodeError: except UnicodeEncodeError:
self.to_screen('[download] The file has already been downloaded') self.to_screen('[download] The file has already been downloaded')
def prepare_filename(self, info_dict): def prepare_filename(self, info_dict, params=None):
"""Generate the output filename.""" """Generate the output filename."""
if params is None:
params = self.params
try: try:
template_dict = dict(info_dict) template_dict = dict(info_dict)
template_dict['epoch'] = int(time.time()) template_dict['epoch'] = int(time.time())
autonumber_size = self.params.get('autonumber_size') autonumber_size = params.get('autonumber_size')
if autonumber_size is None: if autonumber_size is None:
autonumber_size = 5 autonumber_size = 5
autonumber_templ = '%0' + str(autonumber_size) + 'd' autonumber_templ = '%0' + str(autonumber_size) + 'd'
@ -579,14 +581,14 @@ class YoutubeDL(object):
sanitize = lambda k, v: sanitize_filename( sanitize = lambda k, v: sanitize_filename(
compat_str(v), compat_str(v),
restricted=self.params.get('restrictfilenames'), restricted=params.get('restrictfilenames'),
is_id=(k == 'id')) is_id=(k == 'id'))
template_dict = dict((k, sanitize(k, v)) template_dict = dict((k, sanitize(k, v))
for k, v in template_dict.items() for k, v in template_dict.items()
if v is not None) if v is not None)
template_dict = collections.defaultdict(lambda: 'NA', template_dict) template_dict = collections.defaultdict(lambda: 'NA', template_dict)
outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) outtmpl = params.get('outtmpl', DEFAULT_OUTTMPL)
tmpl = compat_expanduser(outtmpl) tmpl = compat_expanduser(outtmpl)
filename = tmpl % template_dict filename = tmpl % template_dict
# Temporary fix for #4787 # Temporary fix for #4787
@ -683,7 +685,9 @@ class YoutubeDL(object):
} }
self.add_default_extra_info(ie_result, ie, url) self.add_default_extra_info(ie_result, ie, url)
if process: if process:
return self.process_ie_result(ie_result, download, extra_info) return self.process_ie_result(
ie_result, download, extra_info,
params=self.params.section(ie.IE_NAME))
else: else:
return ie_result return ie_result
except ExtractorError as e: # An error we somewhat expected except ExtractorError as e: # An error we somewhat expected
@ -708,7 +712,7 @@ class YoutubeDL(object):
'extractor_key': ie.ie_key(), 'extractor_key': ie.ie_key(),
}) })
def process_ie_result(self, ie_result, download=True, extra_info={}): def process_ie_result(self, ie_result, download=True, extra_info={}, params=None):
""" """
Take the result of the ie(may be modified) and resolve all unresolved Take the result of the ie(may be modified) and resolve all unresolved
references (URLs, playlist items). references (URLs, playlist items).
@ -716,6 +720,8 @@ class YoutubeDL(object):
It will also download the videos if 'download'. It will also download the videos if 'download'.
Returns the resolved ie_result. Returns the resolved ie_result.
""" """
if params is None:
params = self.params
result_type = ie_result.get('_type', 'video') result_type = ie_result.get('_type', 'video')
if result_type in ('url', 'url_transparent'): if result_type in ('url', 'url_transparent'):
@ -728,7 +734,7 @@ class YoutubeDL(object):
if result_type == 'video': if result_type == 'video':
self.add_extra_info(ie_result, extra_info) self.add_extra_info(ie_result, extra_info)
return self.process_video_result(ie_result, download=download) return self.process_video_result(ie_result, params, download=download)
elif result_type == 'url': elif result_type == 'url':
# We have to add extra_info to the results because it may be # We have to add extra_info to the results because it may be
# contained in a playlist # contained in a playlist
@ -753,7 +759,7 @@ class YoutubeDL(object):
assert new_result.get('_type') != 'url_transparent' assert new_result.get('_type') != 'url_transparent'
return self.process_ie_result( return self.process_ie_result(
new_result, download=download, extra_info=extra_info) new_result, download=download, extra_info=extra_info, params=params)
elif result_type == 'playlist' or result_type == 'multi_video': elif result_type == 'playlist' or result_type == 'multi_video':
# We process each entry in the playlist # We process each entry in the playlist
playlist = ie_result.get('title') or ie_result.get('id') playlist = ie_result.get('title') or ie_result.get('id')
@ -843,7 +849,8 @@ class YoutubeDL(object):
entry_result = self.process_ie_result(entry, entry_result = self.process_ie_result(entry,
download=download, download=download,
extra_info=extra) extra_info=extra,
params=params)
playlist_results.append(entry_result) playlist_results.append(entry_result)
ie_result['entries'] = playlist_results ie_result['entries'] = playlist_results
self.to_screen('[download] Finished downloading playlist: %s' % playlist) self.to_screen('[download] Finished downloading playlist: %s' % playlist)
@ -865,7 +872,7 @@ class YoutubeDL(object):
) )
return r return r
ie_result['entries'] = [ ie_result['entries'] = [
self.process_ie_result(_fixup(r), download, extra_info) self.process_ie_result(_fixup(r), download, extra_info, params=params)
for r in ie_result['entries'] for r in ie_result['entries']
] ]
return ie_result return ie_result
@ -1213,7 +1220,7 @@ class YoutubeDL(object):
self.cookiejar.add_cookie_header(pr) self.cookiejar.add_cookie_header(pr)
return pr.get_header('Cookie') return pr.get_header('Cookie')
def process_video_result(self, info_dict, download=True): def process_video_result(self, info_dict, params, download=True):
assert info_dict.get('_type', 'video') == 'video' assert info_dict.get('_type', 'video') == 'video'
if 'id' not in info_dict: if 'id' not in info_dict:
@ -1283,7 +1290,7 @@ class YoutubeDL(object):
return return
info_dict['requested_subtitles'] = self.process_subtitles( info_dict['requested_subtitles'] = self.process_subtitles(
info_dict['id'], subtitles, info_dict['id'], subtitles,
info_dict.get('automatic_captions')) info_dict.get('automatic_captions'), params)
# We now pick which formats have to be downloaded # We now pick which formats have to be downloaded
if info_dict.get('formats') is None: if info_dict.get('formats') is None:
@ -1352,10 +1359,10 @@ class YoutubeDL(object):
self.list_formats(info_dict) self.list_formats(info_dict)
return return
req_format = self.params.get('format') req_format = params.get('format')
if req_format is None: if req_format is None:
req_format_list = [] req_format_list = []
if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and if (params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
not info_dict.get('is_live')): not info_dict.get('is_live')):
merger = FFmpegMergerPP(self) merger = FFmpegMergerPP(self)
if merger.available and merger.can_merge(): if merger.available and merger.can_merge():
@ -1374,37 +1381,37 @@ class YoutubeDL(object):
for format in formats_to_download: for format in formats_to_download:
new_info = dict(info_dict) new_info = dict(info_dict)
new_info.update(format) new_info.update(format)
self.process_info(new_info) self.process_info(new_info, params)
# We update the info dict with the best quality format (backwards compatibility) # We update the info dict with the best quality format (backwards compatibility)
info_dict.update(formats_to_download[-1]) info_dict.update(formats_to_download[-1])
return info_dict return info_dict
def process_subtitles(self, video_id, normal_subtitles, automatic_captions): def process_subtitles(self, video_id, normal_subtitles, automatic_captions, params):
"""Select the requested subtitles and their format""" """Select the requested subtitles and their format"""
available_subs = {} available_subs = {}
if normal_subtitles and self.params.get('writesubtitles'): if normal_subtitles and params.get('writesubtitles'):
available_subs.update(normal_subtitles) available_subs.update(normal_subtitles)
if automatic_captions and self.params.get('writeautomaticsub'): if automatic_captions and params.get('writeautomaticsub'):
for lang, cap_info in automatic_captions.items(): for lang, cap_info in automatic_captions.items():
if lang not in available_subs: if lang not in available_subs:
available_subs[lang] = cap_info available_subs[lang] = cap_info
if (not self.params.get('writesubtitles') and not if (not params.get('writesubtitles') and not
self.params.get('writeautomaticsub') or not params.get('writeautomaticsub') or not
available_subs): available_subs):
return None return None
if self.params.get('allsubtitles', False): if params.get('allsubtitles', False):
requested_langs = available_subs.keys() requested_langs = available_subs.keys()
else: else:
if self.params.get('subtitleslangs', False): if params.get('subtitleslangs', False):
requested_langs = self.params.get('subtitleslangs') requested_langs = params.get('subtitleslangs')
elif 'en' in available_subs: elif 'en' in available_subs:
requested_langs = ['en'] requested_langs = ['en']
else: else:
requested_langs = [list(available_subs.keys())[0]] requested_langs = [list(available_subs.keys())[0]]
formats_query = self.params.get('subtitlesformat', 'best') formats_query = params.get('subtitlesformat', 'best')
formats_preference = formats_query.split('/') if formats_query else [] formats_preference = formats_query.split('/') if formats_query else []
subs = {} subs = {}
for lang in requested_langs: for lang in requested_langs:
@ -1428,7 +1435,7 @@ class YoutubeDL(object):
subs[lang] = f subs[lang] = f
return subs return subs
def process_info(self, info_dict): def process_info(self, info_dict, params):
"""Process a single resolved IE result.""" """Process a single resolved IE result."""
assert info_dict.get('_type', 'video') == 'video' assert info_dict.get('_type', 'video') == 'video'
@ -1452,7 +1459,7 @@ class YoutubeDL(object):
self._num_downloads += 1 self._num_downloads += 1
info_dict['_filename'] = filename = self.prepare_filename(info_dict) info_dict['_filename'] = filename = self.prepare_filename(info_dict, params)
# Forced printings # Forced printings
if self.params.get('forcetitle', False): if self.params.get('forcetitle', False):
@ -1494,9 +1501,9 @@ class YoutubeDL(object):
self.report_error('unable to create directory ' + error_to_compat_str(err)) self.report_error('unable to create directory ' + error_to_compat_str(err))
return return
if self.params.get('writedescription', False): if params.get('writedescription', False):
descfn = replace_extension(filename, 'description', info_dict.get('ext')) descfn = replace_extension(filename, 'description', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): if params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
self.to_screen('[info] Video description is already present') self.to_screen('[info] Video description is already present')
elif info_dict.get('description') is None: elif info_dict.get('description') is None:
self.report_warning('There\'s no description to write.') self.report_warning('There\'s no description to write.')
@ -1509,9 +1516,9 @@ class YoutubeDL(object):
self.report_error('Cannot write description file ' + descfn) self.report_error('Cannot write description file ' + descfn)
return return
if self.params.get('writeannotations', False): if params.get('writeannotations', False):
annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext')) annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): if params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
self.to_screen('[info] Video annotations are already present') self.to_screen('[info] Video annotations are already present')
else: else:
try: try:
@ -1524,8 +1531,8 @@ class YoutubeDL(object):
self.report_error('Cannot write annotations file: ' + annofn) self.report_error('Cannot write annotations file: ' + annofn)
return return
subtitles_are_requested = any([self.params.get('writesubtitles', False), subtitles_are_requested = any([params.get('writesubtitles', False),
self.params.get('writeautomaticsub')]) params.get('writeautomaticsub')])
if subtitles_are_requested and info_dict.get('requested_subtitles'): if subtitles_are_requested and info_dict.get('requested_subtitles'):
# subtitles download errors are already managed as troubles in relevant IE # subtitles download errors are already managed as troubles in relevant IE
@ -1546,7 +1553,7 @@ class YoutubeDL(object):
continue continue
try: try:
sub_filename = subtitles_filename(filename, sub_lang, sub_format) sub_filename = subtitles_filename(filename, sub_lang, sub_format)
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): if params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
else: else:
self.to_screen('[info] Writing video subtitles to: ' + sub_filename) self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@ -1556,9 +1563,9 @@ class YoutubeDL(object):
self.report_error('Cannot write subtitles file ' + sub_filename) self.report_error('Cannot write subtitles file ' + sub_filename)
return return
if self.params.get('writeinfojson', False): if params.get('writeinfojson', False):
infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): if params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
self.to_screen('[info] Video description metadata is already present') self.to_screen('[info] Video description metadata is already present')
else: else:
self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn) self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
@ -1570,10 +1577,10 @@ class YoutubeDL(object):
self._write_thumbnails(info_dict, filename) self._write_thumbnails(info_dict, filename)
if not self.params.get('skip_download', False): if not params.get('skip_download', False):
try: try:
def dl(name, info): def dl(name, info):
fd = get_suitable_downloader(info, self.params)(self, self.params) fd = get_suitable_downloader(info, params)(self, params)
for ph in self._progress_hooks: for ph in self._progress_hooks:
fd.add_progress_hook(ph) fd.add_progress_hook(ph)
if self.params.get('verbose'): if self.params.get('verbose'):
@ -1613,7 +1620,7 @@ class YoutubeDL(object):
if filename_real_ext == info_dict['ext'] if filename_real_ext == info_dict['ext']
else filename) else filename)
requested_formats = info_dict['requested_formats'] requested_formats = info_dict['requested_formats']
if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats): if params.get('merge_output_format') is None and not compatible_formats(requested_formats):
info_dict['ext'] = 'mkv' info_dict['ext'] = 'mkv'
self.report_warning( self.report_warning(
'Requested formats are incompatible for merge and will be merged into mkv.') 'Requested formats are incompatible for merge and will be merged into mkv.')
@ -1627,7 +1634,7 @@ class YoutubeDL(object):
for f in requested_formats: for f in requested_formats:
new_info = dict(info_dict) new_info = dict(info_dict)
new_info.update(f) new_info.update(f)
fname = self.prepare_filename(new_info) fname = self.prepare_filename(new_info, params)
fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext']) fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
downloaded.append(fname) downloaded.append(fname)
partial_success = dl(fname, new_info) partial_success = dl(fname, new_info)
@ -1648,7 +1655,7 @@ class YoutubeDL(object):
if success and filename != '-': if success and filename != '-':
# Fixup content # Fixup content
fixup_policy = self.params.get('fixup') fixup_policy = params.get('fixup')
if fixup_policy is None: if fixup_policy is None:
fixup_policy = 'detect_or_warn' fixup_policy = 'detect_or_warn'
@ -1693,7 +1700,7 @@ class YoutubeDL(object):
if (info_dict.get('protocol') == 'm3u8_native' or if (info_dict.get('protocol') == 'm3u8_native' or
info_dict.get('protocol') == 'm3u8' and info_dict.get('protocol') == 'm3u8' and
self.params.get('hls_prefer_native')): params.get('hls_prefer_native')):
if fixup_policy == 'warn': if fixup_policy == 'warn':
self.report_warning('%s: malformated aac bitstream.' % ( self.report_warning('%s: malformated aac bitstream.' % (
info_dict['id'])) info_dict['id']))
@ -1710,7 +1717,7 @@ class YoutubeDL(object):
assert fixup_policy in ('ignore', 'never') assert fixup_policy in ('ignore', 'never')
try: try:
self.post_process(filename, info_dict) self.post_process(filename, info_dict, params)
except (PostProcessingError) as err: except (PostProcessingError) as err:
self.report_error('postprocessing: %s' % str(err)) self.report_error('postprocessing: %s' % str(err))
return return
@ -1747,7 +1754,11 @@ class YoutubeDL(object):
# FileInput doesn't have a read method, we can't call json.load # FileInput doesn't have a read method, we can't call json.load
info = self.filter_requested_info(json.loads('\n'.join(f))) info = self.filter_requested_info(json.loads('\n'.join(f)))
try: try:
self.process_ie_result(info, download=True) params = None
ie_name = info.get('extractor')
if ie_name:
params = self.params.section(ie_name)
self.process_ie_result(info, download=True, params=params)
except DownloadError: except DownloadError:
webpage_url = info.get('webpage_url') webpage_url = info.get('webpage_url')
if webpage_url is not None: if webpage_url is not None:
@ -1763,7 +1774,7 @@ class YoutubeDL(object):
(k, v) for k, v in info_dict.items() (k, v) for k, v in info_dict.items()
if k not in ['requested_formats', 'requested_subtitles']) if k not in ['requested_formats', 'requested_subtitles'])
def post_process(self, filename, ie_info): def post_process(self, filename, ie_info, params):
"""Run all the postprocessors on the given file.""" """Run all the postprocessors on the given file."""
info = dict(ie_info) info = dict(ie_info)
info['filepath'] = filename info['filepath'] = filename
@ -1777,7 +1788,7 @@ class YoutubeDL(object):
files_to_delete, info = pp.run(info) files_to_delete, info = pp.run(info)
except PostProcessingError as e: except PostProcessingError as e:
self.report_error(e.msg) self.report_error(e.msg)
if files_to_delete and not self.params.get('keepvideo', False): if files_to_delete and not params.get('keepvideo', False):
for old_filename in files_to_delete: for old_filename in files_to_delete:
self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
try: try: