From 60dca1b0c035bc356bf914497f212fe9fd619bca Mon Sep 17 00:00:00 2001 From: Logan Fleur Date: Fri, 1 Jun 2018 07:11:56 +0200 Subject: [PATCH 1/3] [DW] Update extractor with /playersources API --- youtube_dl/extractor/dw.py | 46 +++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py index d740652f1..1775bfa14 100644 --- a/youtube_dl/extractor/dw.py +++ b/youtube_dl/extractor/dw.py @@ -5,6 +5,7 @@ from .common import InfoExtractor from ..utils import ( int_or_none, unified_strdate, + ExtractorError ) from ..compat import compat_urlparse @@ -12,6 +13,7 @@ from ..compat import compat_urlparse class DWIE(InfoExtractor): IE_NAME = 'dw' _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+(?:av|e)-(?P\d+)' + _SOURCES_URL = 'http://www.dw.com/playersources/%s-%s' _TESTS = [{ # video 'url': 'http://www.dw.com/en/intelligent-light/av-19112290', @@ -21,7 +23,18 @@ class DWIE(InfoExtractor): 'ext': 'mp4', 'title': 'Intelligent light', 'description': 'md5:90e00d5881719f2a6a5827cb74985af1', - 'upload_date': '20160311', + 'upload_date': '20160605', + } + }, { + # 720 video + 'url': 'http://www.dw.com/en/hamburg-imposes-germanys-first-diesel-ban/av-44026376', + 'md5': '09299d024c4003145247d02b45a2bd3f', + 'info_dict': { + 'id': '44026376', + 'ext': 'mp4', + 'title': 'Hamburg imposes Germany\'s first diesel ban', + 'description': 'md5:a455cf13a29a11d5abff2c9a57981198', + 'upload_date': '20180531', } }, { # audio @@ -55,13 +68,30 @@ class DWIE(InfoExtractor): title = hidden_inputs['media_title'] media_id = hidden_inputs.get('media_id') or media_id - if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1': - formats = self._extract_smil_formats( - 'http://www.dw.com/smil/v-%s' % media_id, media_id, - transform_source=lambda s: s.replace( - 'rtmp://tv-od.dw.de/flash/', - 'http://tv-download.dw.de/dwtv_video/flv/')) - self._sort_formats(formats) + if hidden_inputs.get('player_type') == 'video': + if hidden_inputs.get('stream_file') == '1': + formats = self._extract_smil_formats( + 'http://www.dw.com/smil/v-%s' % media_id, media_id, + transform_source=lambda s: s.replace( + 'rtmp://tv-od.dw.de/flash/', + 'http://tv-download.dw.de/dwtv_video/flv/')) + self._sort_formats(formats) + else: + media = self._download_json(self._SOURCES_URL % ('v', media_id), media_id) + formats = [] + for medium in media: + formats.append({ + 'url': medium.get('file'), + 'format_id': medium.get('label') + }) + self._sort_formats(formats, 'label') + elif hidden_inputs.get('player_type') == 'audio': + media = self._download_json(self._SOURCES_URL % ('a', media_id), media_id) + formats = [] + for medium in media: + formats.append({ + 'url': medium.get('file'), + }) else: formats = [{'url': hidden_inputs['file_name']}] From 744e369543031bd758297161e2fbd94ed42c2edc Mon Sep 17 00:00:00 2001 From: Logan Fleur Date: Mon, 9 Jul 2018 21:45:50 +0200 Subject: [PATCH 2/3] Removed duplicate code/calls --- youtube_dl/extractor/dw.py | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py index 1775bfa14..e212598ed 100644 --- a/youtube_dl/extractor/dw.py +++ b/youtube_dl/extractor/dw.py @@ -5,7 +5,6 @@ from .common import InfoExtractor from ..utils import ( int_or_none, unified_strdate, - ExtractorError ) from ..compat import compat_urlparse @@ -62,13 +61,27 @@ class DWIE(InfoExtractor): }] def _real_extract(self, url): + + def extract_formats(): + media = self._download_json(self._SOURCES_URL % (player_type[0], media_id), media_id) + extracted_formats = [] + for medium in media: + extracted_formats.append({ + 'url': medium.get('file') + }) + label = medium.get('label') + if label is not None: + extracted_formats[-1]['format_id'] = label + return extracted_formats + media_id = self._match_id(url) webpage = self._download_webpage(url, media_id) hidden_inputs = self._hidden_inputs(webpage) - title = hidden_inputs['media_title'] + title = hidden_inputs.get('media_title') media_id = hidden_inputs.get('media_id') or media_id + player_type = hidden_inputs.get('player_type') - if hidden_inputs.get('player_type') == 'video': + if player_type == 'video': if hidden_inputs.get('stream_file') == '1': formats = self._extract_smil_formats( 'http://www.dw.com/smil/v-%s' % media_id, media_id, @@ -77,23 +90,12 @@ class DWIE(InfoExtractor): 'http://tv-download.dw.de/dwtv_video/flv/')) self._sort_formats(formats) else: - media = self._download_json(self._SOURCES_URL % ('v', media_id), media_id) - formats = [] - for medium in media: - formats.append({ - 'url': medium.get('file'), - 'format_id': medium.get('label') - }) - self._sort_formats(formats, 'label') - elif hidden_inputs.get('player_type') == 'audio': - media = self._download_json(self._SOURCES_URL % ('a', media_id), media_id) - formats = [] - for medium in media: - formats.append({ - 'url': medium.get('file'), - }) + formats = extract_formats() + self._sort_formats(formats) + elif player_type == 'audio': + formats = extract_formats() else: - formats = [{'url': hidden_inputs['file_name']}] + formats = [{'url': hidden_inputs.get('file_name')}] upload_date = hidden_inputs.get('display_date') if not upload_date: From 6e55c2c2646e9c91c406f7a145663f76b64c7113 Mon Sep 17 00:00:00 2001 From: Logan Fleur Date: Mon, 9 Jul 2018 22:58:15 +0200 Subject: [PATCH 3/3] Removed possibility of None as format URL --- youtube_dl/extractor/dw.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py index e212598ed..49aa9c29a 100644 --- a/youtube_dl/extractor/dw.py +++ b/youtube_dl/extractor/dw.py @@ -66,12 +66,14 @@ class DWIE(InfoExtractor): media = self._download_json(self._SOURCES_URL % (player_type[0], media_id), media_id) extracted_formats = [] for medium in media: - extracted_formats.append({ - 'url': medium.get('file') - }) - label = medium.get('label') - if label is not None: - extracted_formats[-1]['format_id'] = label + file = medium.get('file') + if file is not None: + extracted_formats.append({ + 'url': file + }) + label = medium.get('label') + if label is not None: + extracted_formats[-1]['format_id'] = label return extracted_formats media_id = self._match_id(url) @@ -80,22 +82,19 @@ class DWIE(InfoExtractor): title = hidden_inputs.get('media_title') media_id = hidden_inputs.get('media_id') or media_id player_type = hidden_inputs.get('player_type') + file_name = hidden_inputs.get('file_name') + formats = [] - if player_type == 'video': - if hidden_inputs.get('stream_file') == '1': + if player_type == 'video' and hidden_inputs.get('stream_file') == '1': formats = self._extract_smil_formats( 'http://www.dw.com/smil/v-%s' % media_id, media_id, transform_source=lambda s: s.replace( 'rtmp://tv-od.dw.de/flash/', 'http://tv-download.dw.de/dwtv_video/flv/')) - self._sort_formats(formats) - else: - formats = extract_formats() - self._sort_formats(formats) - elif player_type == 'audio': - formats = extract_formats() - else: - formats = [{'url': hidden_inputs.get('file_name')}] + + if not formats: + formats = extract_formats() or [{'url': file_name}] if file_name else [] + self._sort_formats(formats) upload_date = hidden_inputs.get('display_date') if not upload_date: