From 8fc226ef994a82f7b1050cdb72ec38922d3ab9cf Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 2 Oct 2015 17:24:30 +0100 Subject: [PATCH 001/110] [nba] extract all video formats and extract more info --- youtube_dl/extractor/__init__.py | 5 +- youtube_dl/extractor/nba.py | 102 +++++++++++++++++++++---------- 2 files changed, 74 insertions(+), 33 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a73a1317e..78478b38b 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -351,7 +351,10 @@ from .myvideo import MyVideoIE from .myvidster import MyVidsterIE from .nationalgeographic import NationalGeographicIE from .naver import NaverIE -from .nba import NBAIE +from .nba import ( + NBAIE, + NBAWatchIE, +) from .nbc import ( NBCIE, NBCNewsIE, diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index 944096e1c..36ece5b64 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -2,62 +2,100 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( - remove_end, parse_duration, + parse_iso8601, + int_or_none, ) -class NBAIE(InfoExtractor): - _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' +class NBABaseIE(InfoExtractor): + def _get_formats(self, video_id): + base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id + return [{ + 'url': base_url + '_nba_android_high.mp4', + 'width': 480, + 'height': 320, + 'format_id': '320p', + },{ + 'url': base_url + '_640x360_664b.mp4', + 'width': 640, + 'height': 360, + 'format_id': '360p', + },{ + 'url': base_url + '_768x432_1404.mp4', + 'width': 768, + 'height': 432, + 'format_id': '432p', + },{ + 'url': base_url + '_1280x720.mp4', + 'width': 1280, + 'height': 720, + 'format_id': '720p', + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + ret = self._extract_metadata(webpage, video_id) + ret['id'] = video_id.rpartition('/')[2] + ret['formats'] = self._get_formats(video_id) + return ret + + +class NBAIE(NBABaseIE): + IE_NAME = 'nba' + _VALID_URL = r'https?://(?:www\.)?nba\.com/(?:nba/)?video(?P/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' _TESTS = [{ 'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html', - 'md5': 'c0edcfc37607344e2ff8f13c378c88a4', + 'md5': '9d902940d2a127af3f7f9d2f3dc79c96', 'info_dict': { 'id': '0021200253-okc-bkn-recap.nba', 'ext': 'mp4', 'title': 'Thunder vs. Nets', 'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.', 'duration': 181, + 'timestamp': 1354680189, + 'upload_date': '20121205', }, }, { 'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/', 'only_matching': True, - }, { + }] + + def _extract_metadata(self, webpage, video_id): + return { + 'title': self._html_search_meta('name', webpage), + 'description': self._html_search_meta('description', webpage), + 'duration': parse_duration(self._html_search_meta('duration', webpage)), + 'thumbnail': self._html_search_meta('thumbnailUrl', webpage), + 'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage)) + } + +class NBAWatchIE(NBABaseIE): + IE_NAME = 'nba:watch' + _VALID_URL = r'https?://watch.nba\.com/(?:nba/)?video(?P/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' + _TESTS = [{ 'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba', + 'md5': 'b2b39b81cf28615ae0c3360a3f9668c4', 'info_dict': { 'id': '0041400301-cle-atl-recap.nba', 'ext': 'mp4', - 'title': 'NBA GAME TIME | Video: Hawks vs. Cavaliers Game 1', + 'title': 'Hawks vs. Cavaliers Game 1', 'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d', 'duration': 228, - }, - 'params': { - 'skip_download': True, + 'timestamp': 1432094400, + 'upload_date': '20150520', } }] - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4' - - shortened_video_id = video_id.rpartition('/')[2] - title = remove_end( - self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com') - - description = self._og_search_description(webpage) - duration_str = self._html_search_meta( - 'duration', webpage, 'duration', default=None) - if not duration_str: - duration_str = self._html_search_regex( - r'Duration:\s*(\d+:\d+)', webpage, 'duration', fatal=False) - duration = parse_duration(duration_str) - + def _extract_metadata(self, webpage, video_id): + program_id = self._search_regex(r'var\s+programId\s*=\s*(\d+);', webpage, 'program id') + metadata = self._download_json( + 'http://smbsolr.cdnak.neulion.com/solr_nbav6/nba/nba/mlt/?wt=json&fl=name,description,image,runtime,releaseDate&q=sequence%3A' + program_id, video_id)['match']['docs'][0] return { - 'id': shortened_video_id, - 'url': video_url, - 'title': title, - 'description': description, - 'duration': duration, + 'title': metadata['name'], + 'description': metadata.get('description'), + 'duration': int_or_none(metadata.get('runtime')), + 'thumbnail': metadata.get('image'), + 'timestamp': parse_iso8601(metadata.get('releaseDate')) } From 28809ab07a8d10f9cafc3d712414c7b355c27166 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sat, 3 Oct 2015 09:47:19 +0100 Subject: [PATCH 002/110] [nba] extract more formats --- youtube_dl/extractor/nba.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index 36ece5b64..8844b61a5 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -10,28 +10,60 @@ from ..utils import ( class NBABaseIE(InfoExtractor): def _get_formats(self, video_id): + formats = self._extract_m3u8_formats( + 'http://nbavod-f.akamaihd.net/i/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/master.m3u8' % video_id, + video_id, + m3u8_id='hls') + formats.extend(self._extract_f4m_formats( + 'http://nbavod-f.akamaihd.net/z/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/manifest.f4m?hdcore=3.4.1.1' % video_id, + video_id, + f4m_id='hds')) base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id - return [{ + formats.extend([{ + 'url': base_url + '_nba_ipad.mp4', + 'width': 400, + 'height': 224, + 'format_id': '224p', + 'preference': 1, + },{ 'url': base_url + '_nba_android_high.mp4', 'width': 480, 'height': 320, 'format_id': '320p', + 'preference': 2, + },{ + 'url': base_url + '_nba_576x324.mp4', + 'width': 576, + 'height': 324, + 'format_id': '324p', + 'preference': 3, },{ 'url': base_url + '_640x360_664b.mp4', 'width': 640, 'height': 360, 'format_id': '360p', + 'preference': 4, },{ 'url': base_url + '_768x432_1404.mp4', 'width': 768, 'height': 432, 'format_id': '432p', + 'preference': 5, + },{ + 'url': base_url + '_960x540_2104.mp4', + 'width': 960, + 'height': 540, + 'format_id': '540p', + 'preference': 6, },{ 'url': base_url + '_1280x720.mp4', 'width': 1280, 'height': 720, 'format_id': '720p', - }] + 'preference': 7, + }]) + self._sort_formats(formats) + return formats def _real_extract(self, url): video_id = self._match_id(url) From c233e6bcc398f9734d7138854978c1cb00fe757f Mon Sep 17 00:00:00 2001 From: remitamine Date: Sat, 3 Oct 2015 12:30:05 +0100 Subject: [PATCH 003/110] [nba] extract video info from xml feed --- youtube_dl/extractor/__init__.py | 5 +- youtube_dl/extractor/nba.py | 224 +++++++++++++++++-------------- 2 files changed, 126 insertions(+), 103 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 78478b38b..a73a1317e 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -351,10 +351,7 @@ from .myvideo import MyVideoIE from .myvidster import MyVidsterIE from .nationalgeographic import NationalGeographicIE from .naver import NaverIE -from .nba import ( - NBAIE, - NBAWatchIE, -) +from .nba import NBAIE from .nbc import ( NBCIE, NBCNewsIE, diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index 8844b61a5..3d38d080e 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -3,131 +3,157 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( parse_duration, - parse_iso8601, int_or_none, ) -class NBABaseIE(InfoExtractor): - def _get_formats(self, video_id): - formats = self._extract_m3u8_formats( - 'http://nbavod-f.akamaihd.net/i/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/master.m3u8' % video_id, - video_id, - m3u8_id='hls') - formats.extend(self._extract_f4m_formats( - 'http://nbavod-f.akamaihd.net/z/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/manifest.f4m?hdcore=3.4.1.1' % video_id, - video_id, - f4m_id='hds')) - base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id - formats.extend([{ - 'url': base_url + '_nba_ipad.mp4', - 'width': 400, - 'height': 224, - 'format_id': '224p', - 'preference': 1, - },{ - 'url': base_url + '_nba_android_high.mp4', - 'width': 480, - 'height': 320, - 'format_id': '320p', - 'preference': 2, - },{ - 'url': base_url + '_nba_576x324.mp4', - 'width': 576, - 'height': 324, - 'format_id': '324p', - 'preference': 3, - },{ - 'url': base_url + '_640x360_664b.mp4', - 'width': 640, - 'height': 360, - 'format_id': '360p', - 'preference': 4, - },{ - 'url': base_url + '_768x432_1404.mp4', - 'width': 768, - 'height': 432, - 'format_id': '432p', - 'preference': 5, - },{ - 'url': base_url + '_960x540_2104.mp4', - 'width': 960, - 'height': 540, - 'format_id': '540p', - 'preference': 6, - },{ - 'url': base_url + '_1280x720.mp4', - 'width': 1280, - 'height': 720, - 'format_id': '720p', - 'preference': 7, - }]) - self._sort_formats(formats) - return formats - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - ret = self._extract_metadata(webpage, video_id) - ret['id'] = video_id.rpartition('/')[2] - ret['formats'] = self._get_formats(video_id) - return ret - - -class NBAIE(NBABaseIE): - IE_NAME = 'nba' - _VALID_URL = r'https?://(?:www\.)?nba\.com/(?:nba/)?video(?P/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' +class NBAIE(InfoExtractor): + _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video/(?P[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' _TESTS = [{ 'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html', 'md5': '9d902940d2a127af3f7f9d2f3dc79c96', 'info_dict': { - 'id': '0021200253-okc-bkn-recap.nba', + 'id': '0021200253-okc-bkn-recap', 'ext': 'mp4', 'title': 'Thunder vs. Nets', 'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.', 'duration': 181, - 'timestamp': 1354680189, - 'upload_date': '20121205', + 'timestamp': 1354638466, + 'upload_date': '20121204', }, }, { 'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/', 'only_matching': True, - }] - - def _extract_metadata(self, webpage, video_id): - return { - 'title': self._html_search_meta('name', webpage), - 'description': self._html_search_meta('description', webpage), - 'duration': parse_duration(self._html_search_meta('duration', webpage)), - 'thumbnail': self._html_search_meta('thumbnailUrl', webpage), - 'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage)) - } - -class NBAWatchIE(NBABaseIE): - IE_NAME = 'nba:watch' - _VALID_URL = r'https?://watch.nba\.com/(?:nba/)?video(?P/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' - _TESTS = [{ + },{ 'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba', 'md5': 'b2b39b81cf28615ae0c3360a3f9668c4', 'info_dict': { - 'id': '0041400301-cle-atl-recap.nba', + 'id': '0041400301-cle-atl-recap', 'ext': 'mp4', 'title': 'Hawks vs. Cavaliers Game 1', 'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d', 'duration': 228, - 'timestamp': 1432094400, + 'timestamp': 1432134543, 'upload_date': '20150520', } }] - def _extract_metadata(self, webpage, video_id): - program_id = self._search_regex(r'var\s+programId\s*=\s*(\d+);', webpage, 'program id') - metadata = self._download_json( - 'http://smbsolr.cdnak.neulion.com/solr_nbav6/nba/nba/mlt/?wt=json&fl=name,description,image,runtime,releaseDate&q=sequence%3A' + program_id, video_id)['match']['docs'][0] + _BASE_PATHS = { + 'turner': 'http://nba.cdn.turner.com/nba/big', + 'akamai': 'http://nbavod-f.akamaihd.net', + } + + _QUALITIES = { + '420mp4': { + 'width': 400, + 'height': 224, + 'preference': 1, + }, + '416x234': { + 'width': 416, + 'height': 234, + 'preference': 2, + }, + '556': { + 'width': 416, + 'height': 234, + 'preference': 3, + }, + '480x320_910': { + 'width': 480, + 'height': 320, + 'preference': 4, + }, + 'nba_576x324': { + 'width': 576, + 'height': 324, + 'preference': 5, + }, + 'nba_640x360': { + 'width': 640, + 'height': 360, + 'preference': 6, + }, + '640x360_664b': { + 'width': 640, + 'height': 360, + 'preference': 7, + }, + '640x360_664m': { + 'width': 640, + 'height': 360, + 'preference': 8, + }, + '768x432_996': { + 'width': 768, + 'height': 432, + 'preference': 9, + }, + '768x432_1404': { + 'width': 768, + 'height': 432, + 'preference': 10, + }, + '960x540_2104': { + 'width': 960, + 'height': 540, + 'preference': 11, + }, + '1280x720_3072': { + 'width': 1280, + 'height': 720, + 'preference': 12, + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id) + video_id = video_info.find('slug').text + title = video_info.find('headline').text + description = video_info.find('description').text + duration = parse_duration(video_info.find('length').text) + timestamp = int_or_none(video_info.find('dateCreated').attrib.get('uts')) + + thumbnails = [] + for image in video_info.find('images'): + thumbnails.append({ + 'id': image.attrib.get('cut'), + 'url': image.text, + 'width': int_or_none(image.attrib.get('width')), + 'height': int_or_none(image.attrib.get('height')), + }) + + formats = [] + for video_file in video_info.find('files').iter('file'): + video_url = video_file.text + if not video_url.startswith('http://'): + if video_url.endswith('.m3u8') or video_url.endswith('.f4m'): + video_url = self._BASE_PATHS['akamai'] + video_url + else: + video_url = self._BASE_PATHS['turner'] + video_url + if video_url.endswith('.m3u8'): + formats.extend(self._extract_m3u8_formats(video_url, video_id)) + elif video_url.endswith('.f4m'): + formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id)) + else: + key = video_file.attrib.get('bitrate') + quality = self._QUALITIES[key] + formats.append({ + 'format_id': key, + 'url': video_url, + 'width': quality['width'], + 'height': quality['height'], + 'preference': quality['preference'], + }) + self._sort_formats(formats) + return { - 'title': metadata['name'], - 'description': metadata.get('description'), - 'duration': int_or_none(metadata.get('runtime')), - 'thumbnail': metadata.get('image'), - 'timestamp': parse_iso8601(metadata.get('releaseDate')) + 'id': video_id, + 'title': title, + 'description': description, + 'duration': duration, + 'timestamp': timestamp, + 'thumbnails': thumbnails, + 'formats': formats, } From 30787f7259c4e6a08f691cc691f14fa0c8fe4b87 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sat, 3 Oct 2015 19:28:48 +0100 Subject: [PATCH 004/110] [cspan] correct the clip info extraction --- youtube_dl/extractor/cspan.py | 58 ++++++++++++++++------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py index fbefd37d0..994e080d5 100644 --- a/youtube_dl/extractor/cspan.py +++ b/youtube_dl/extractor/cspan.py @@ -18,22 +18,21 @@ class CSpanIE(InfoExtractor): IE_DESC = 'C-SPAN' _TESTS = [{ 'url': 'http://www.c-span.org/video/?313572-1/HolderonV', - 'md5': '8e44ce11f0f725527daccc453f553eb0', + 'md5': '067803f994e049b455a58b16e5aab442', 'info_dict': { 'id': '315139', 'ext': 'mp4', 'title': 'Attorney General Eric Holder on Voting Rights Act Decision', - 'description': 'Attorney General Eric Holder spoke to reporters following the Supreme Court decision in Shelby County v. Holder in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced until Congress established new guidelines for review.', + 'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.', }, 'skip': 'Regularly fails on travis, for unknown reasons', }, { 'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models', - # For whatever reason, the served video alternates between - # two different ones + 'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2', 'info_dict': { - 'id': '340723', + 'id': 'c4486943', 'ext': 'mp4', - 'title': 'International Health Care Models', + 'title': 'CSPAN - International Health Care Models', 'description': 'md5:7a985a2d595dba00af3d9c9f0783c967', } }, { @@ -44,7 +43,7 @@ class CSpanIE(InfoExtractor): 'ext': 'mp4', 'title': 'General Motors Ignition Switch Recall', 'duration': 14848, - 'description': 'md5:70c7c3b8fa63fa60d42772440596034c' + 'description': 'md5:118081aedd24bf1d3b68b3803344e7f3' }, }, { # Video from senate.gov @@ -57,36 +56,33 @@ class CSpanIE(InfoExtractor): }] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - page_id = mobj.group('id') - webpage = self._download_webpage(url, page_id) - video_id = self._search_regex(r'progid=\'?([0-9]+)\'?>', webpage, 'video id') + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage) + if matches: + video_type, video_id = matches.groups() + if video_type == 'prog': + video_type = 'program' + else: + senate_isvp_url = SenateISVPIE._search_iframe_url(webpage) + if senate_isvp_url: + title = self._og_search_title(webpage) + surl = smuggle_url(senate_isvp_url, {'force_title': title}) + return self.url_result(surl, 'SenateISVP', video_id, title) - description = self._html_search_regex( - [ - # The full description - r'
(.*?)(.*?)

' - ], - webpage, 'description', flags=re.DOTALL, default=None) - - info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id - data = self._download_json(info_url, video_id) + data = self._download_json( + 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id), + video_id) doc = self._download_xml( - 'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id, + 'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id), video_id) + description = self._html_search_meta('description', webpage) + title = find_xpath_attr(doc, './/string', 'name', 'title').text thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text - senate_isvp_url = SenateISVPIE._search_iframe_url(webpage) - if senate_isvp_url: - surl = smuggle_url(senate_isvp_url, {'force_title': title}) - return self.url_result(surl, 'SenateISVP', video_id, title) - files = data['video']['files'] try: capfile = data['video']['capfile']['#text'] @@ -112,12 +108,12 @@ class CSpanIE(InfoExtractor): if len(entries) == 1: entry = dict(entries[0]) - entry['id'] = video_id + entry['id'] = 'c' + video_id if video_type == 'clip' else video_id return entry else: return { '_type': 'playlist', 'entries': entries, 'title': title, - 'id': video_id, + 'id': 'c' + video_id if video_type == 'clip' else video_id, } From 139f27827e1d771aba5cf7f1473129073686f5ab Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 7 Oct 2015 06:53:19 +0100 Subject: [PATCH 005/110] [nba] skip Legacy Video Files --- youtube_dl/extractor/nba.py | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index 3d38d080e..73116c7c6 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -38,11 +38,6 @@ class NBAIE(InfoExtractor): } }] - _BASE_PATHS = { - 'turner': 'http://nba.cdn.turner.com/nba/big', - 'akamai': 'http://nbavod-f.akamaihd.net', - } - _QUALITIES = { '420mp4': { 'width': 400, @@ -54,55 +49,50 @@ class NBAIE(InfoExtractor): 'height': 234, 'preference': 2, }, - '556': { - 'width': 416, - 'height': 234, - 'preference': 3, - }, '480x320_910': { 'width': 480, 'height': 320, - 'preference': 4, + 'preference': 3, }, 'nba_576x324': { 'width': 576, 'height': 324, - 'preference': 5, + 'preference': 4, }, 'nba_640x360': { 'width': 640, 'height': 360, - 'preference': 6, + 'preference': 5, }, '640x360_664b': { 'width': 640, 'height': 360, - 'preference': 7, + 'preference': 6, }, '640x360_664m': { 'width': 640, 'height': 360, - 'preference': 8, + 'preference': 7, }, '768x432_996': { 'width': 768, 'height': 432, - 'preference': 9, + 'preference': 8, }, '768x432_1404': { 'width': 768, 'height': 432, - 'preference': 10, + 'preference': 9, }, '960x540_2104': { 'width': 960, 'height': 540, - 'preference': 11, + 'preference': 10, }, '1280x720_3072': { 'width': 1280, 'height': 720, - 'preference': 12, + 'preference': 11, }, } @@ -127,11 +117,8 @@ class NBAIE(InfoExtractor): formats = [] for video_file in video_info.find('files').iter('file'): video_url = video_file.text - if not video_url.startswith('http://'): - if video_url.endswith('.m3u8') or video_url.endswith('.f4m'): - video_url = self._BASE_PATHS['akamai'] + video_url - else: - video_url = self._BASE_PATHS['turner'] + video_url + if video_url.startswith('/'): + continue if video_url.endswith('.m3u8'): formats.extend(self._extract_m3u8_formats(video_url, video_id)) elif video_url.endswith('.f4m'): From ecf6de5b02ad3996f770efd33f9b400d04ac8a85 Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 7 Oct 2015 07:09:45 +0100 Subject: [PATCH 006/110] [nba] extract width,height and bitrate from format key --- youtube_dl/extractor/nba.py | 68 ++++--------------------------------- 1 file changed, 6 insertions(+), 62 deletions(-) diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index 73116c7c6..ea1482fc8 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import ( parse_duration, @@ -38,64 +40,6 @@ class NBAIE(InfoExtractor): } }] - _QUALITIES = { - '420mp4': { - 'width': 400, - 'height': 224, - 'preference': 1, - }, - '416x234': { - 'width': 416, - 'height': 234, - 'preference': 2, - }, - '480x320_910': { - 'width': 480, - 'height': 320, - 'preference': 3, - }, - 'nba_576x324': { - 'width': 576, - 'height': 324, - 'preference': 4, - }, - 'nba_640x360': { - 'width': 640, - 'height': 360, - 'preference': 5, - }, - '640x360_664b': { - 'width': 640, - 'height': 360, - 'preference': 6, - }, - '640x360_664m': { - 'width': 640, - 'height': 360, - 'preference': 7, - }, - '768x432_996': { - 'width': 768, - 'height': 432, - 'preference': 8, - }, - '768x432_1404': { - 'width': 768, - 'height': 432, - 'preference': 9, - }, - '960x540_2104': { - 'width': 960, - 'height': 540, - 'preference': 10, - }, - '1280x720_3072': { - 'width': 1280, - 'height': 720, - 'preference': 11, - }, - } - def _real_extract(self, url): video_id = self._match_id(url) video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id) @@ -125,13 +69,13 @@ class NBAIE(InfoExtractor): formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id)) else: key = video_file.attrib.get('bitrate') - quality = self._QUALITIES[key] + width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups() formats.append({ 'format_id': key, 'url': video_url, - 'width': quality['width'], - 'height': quality['height'], - 'preference': quality['preference'], + 'width': int_or_none(width), + 'height': int_or_none(height), + 'tbr': int_or_none(bitrate), }) self._sort_formats(formats) From 6a11bb77baf9f70da76f2595b74061b31223d4ff Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 7 Oct 2015 12:17:32 +0100 Subject: [PATCH 007/110] [nba] add support for team subsites --- youtube_dl/extractor/nba.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py index ea1482fc8..a0cc58c12 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dl/extractor/nba.py @@ -10,13 +10,13 @@ from ..utils import ( class NBAIE(InfoExtractor): - _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video/(?P[^?]*?)/?(?:/index\.html)?(?:\?.*)?$' + _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P(?:[^/]+/)?video/(?P[^?]*?))/?(?:/index\.html)?(?:\?.*)?$' _TESTS = [{ 'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html', - 'md5': '9d902940d2a127af3f7f9d2f3dc79c96', + 'md5': '9e7729d3010a9c71506fd1248f74e4f4', 'info_dict': { 'id': '0021200253-okc-bkn-recap', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'Thunder vs. Nets', 'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.', 'duration': 181, @@ -27,7 +27,7 @@ class NBAIE(InfoExtractor): 'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/', 'only_matching': True, },{ - 'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba', + 'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba', 'md5': 'b2b39b81cf28615ae0c3360a3f9668c4', 'info_dict': { 'id': '0041400301-cle-atl-recap', @@ -41,8 +41,8 @@ class NBAIE(InfoExtractor): }] def _real_extract(self, url): - video_id = self._match_id(url) - video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id) + path, video_id = re.match(self._VALID_URL, url).groups() + video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id) video_id = video_info.find('slug').text title = video_info.find('headline').text description = video_info.find('description').text @@ -64,9 +64,9 @@ class NBAIE(InfoExtractor): if video_url.startswith('/'): continue if video_url.endswith('.m3u8'): - formats.extend(self._extract_m3u8_formats(video_url, video_id)) + formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls')) elif video_url.endswith('.f4m'): - formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id)) + formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds')) else: key = video_file.attrib.get('bitrate') width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups() From 90bddb6cdd59107d137c13970dc50a6193d204a7 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 15 Oct 2015 14:28:56 +0100 Subject: [PATCH 008/110] [ooyala] extract more formats and metadata --- youtube_dl/extractor/ooyala.py | 151 ++++++++++++--------------------- 1 file changed, 53 insertions(+), 98 deletions(-) diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index a262a9f6d..592cdc564 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -1,108 +1,64 @@ from __future__ import unicode_literals import re -import json import base64 from .common import InfoExtractor from ..utils import ( - unescapeHTML, - ExtractorError, - determine_ext, int_or_none, + float_or_none, ) class OoyalaBaseIE(InfoExtractor): - def _extract_result(self, info, more_info): - embedCode = info['embedCode'] - video_url = info.get('ipad_url') or info['url'] - - if determine_ext(video_url) == 'm3u8': - formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4') - else: - formats = [{ - 'url': video_url, - 'ext': 'mp4', - }] - - return { - 'id': embedCode, - 'title': unescapeHTML(info['title']), - 'formats': formats, - 'description': unescapeHTML(more_info['description']), - 'thumbnail': more_info['promo'], + def _extract(self, player_url, video_id): + print(player_url) + content_tree = self._download_json(player_url, video_id)['content_tree'] + metadata = content_tree[list(content_tree)[0]] + embed_code = metadata['embed_code'] + pcode = metadata.get('asset_pcode') or embed_code + video_info = { + 'id': embed_code, + 'title': metadata['title'], + 'description': metadata.get('description'), + 'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'), + 'duration': int_or_none(metadata.get('duration')), } - def _extract(self, player_url, video_id): - player = self._download_webpage(player_url, video_id) - mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="', - player, 'mobile player url') - # Looks like some videos are only available for particular devices - # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0 - # is only available for ipad) - # Working around with fetching URLs for all the devices found starting with 'unknown' - # until we succeed or eventually fail for each device. - devices = re.findall(r'device\s*=\s*"([^"]+)";', player) - devices.remove('unknown') - devices.insert(0, 'unknown') - for device in devices: - mobile_player = self._download_webpage( - '%s&device=%s' % (mobile_url, device), video_id, - 'Downloading mobile player JS for %s device' % device) - videos_info = self._search_regex( - r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);', - mobile_player, 'info', fatal=False, default=None) - if videos_info: - break - - if not videos_info: - formats = [] + formats = [] + for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'): auth_data = self._download_json( - 'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (video_id, video_id), - video_id) + 'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=%s' % (pcode, embed_code, supported_format), + video_id, 'Downloading %s JSON' % supported_format) - cur_auth_data = auth_data['authorization_data'][video_id] + cur_auth_data = auth_data['authorization_data'][embed_code] for stream in cur_auth_data['streams']: - formats.append({ - 'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'), - 'ext': stream.get('delivery_type'), - 'format': stream.get('video_codec'), - 'format_id': stream.get('profile'), - 'width': int_or_none(stream.get('width')), - 'height': int_or_none(stream.get('height')), - 'abr': int_or_none(stream.get('audio_bitrate')), - 'vbr': int_or_none(stream.get('video_bitrate')), - }) - if formats: - return { - 'id': video_id, - 'formats': formats, - 'title': 'Ooyala video', - } + url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8') + delivery_type = stream['delivery_type'] + if delivery_type == 'remote_asset': + video_info['url'] = url + return video_info + if delivery_type == 'hls': + formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', 0, m3u8_id='hls', fatal=False)) + elif delivery_type == 'hds': + formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False)) + else: + formats.append({ + 'url': url, + 'ext': stream.get('delivery_type'), + 'vcodec': stream.get('video_codec'), + 'format_id': stream.get('profile'), + 'width': int_or_none(stream.get('width')), + 'height': int_or_none(stream.get('height')), + 'abr': int_or_none(stream.get('audio_bitrate')), + 'vbr': int_or_none(stream.get('video_bitrate')), + 'fps': float_or_none(stream.get('framerate')), + }) + self._sort_formats(formats) - if not cur_auth_data['authorized']: - raise ExtractorError(cur_auth_data['message'], expected=True) - - if not videos_info: - raise ExtractorError('Unable to extract info') - videos_info = videos_info.replace('\\"', '"') - videos_more_info = self._search_regex( - r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"') - videos_info = json.loads(videos_info) - videos_more_info = json.loads(videos_more_info) - - if videos_more_info.get('lineup'): - videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])] - return { - '_type': 'playlist', - 'id': video_id, - 'title': unescapeHTML(videos_more_info['title']), - 'entries': videos, - } - else: - return self._extract_result(videos_info[0], videos_more_info) + video_info['formats'] = formats + return video_info class OoyalaIE(OoyalaBaseIE): @@ -117,6 +73,7 @@ class OoyalaIE(OoyalaBaseIE): 'ext': 'mp4', 'title': 'Explaining Data Recovery from Hard Drives and SSDs', 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.', + 'duration': 853386, }, }, { # Only available for ipad @@ -125,7 +82,7 @@ class OoyalaIE(OoyalaBaseIE): 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0', 'ext': 'mp4', 'title': 'Simulation Overview - Levels of Simulation', - 'description': '', + 'duration': 194948, }, }, { @@ -136,7 +93,8 @@ class OoyalaIE(OoyalaBaseIE): 'info_dict': { 'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx', 'ext': 'mp4', - 'title': 'Ooyala video', + 'title': 'Divide Tool Path.mp4', + 'duration': 204405, } } ] @@ -152,8 +110,8 @@ class OoyalaIE(OoyalaBaseIE): def _real_extract(self, url): embed_code = self._match_id(url) - player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code - return self._extract(player_url, embed_code) + content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code) + return self._extract(content_tree_url, embed_code) class OoyalaExternalIE(OoyalaBaseIE): @@ -170,7 +128,7 @@ class OoyalaExternalIE(OoyalaBaseIE): .*?&pcode= ) (?P.+?) - (&|$) + (?:&|$) ''' _TEST = { @@ -179,7 +137,7 @@ class OoyalaExternalIE(OoyalaBaseIE): 'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG', 'ext': 'mp4', 'title': 'dm_140128_30for30Shorts___JudgingJewellv2', - 'description': '', + 'duration': 1302000, }, 'params': { # m3u8 download @@ -188,9 +146,6 @@ class OoyalaExternalIE(OoyalaBaseIE): } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - partner_id = mobj.group('partner_id') - video_id = mobj.group('id') - pcode = mobj.group('pcode') - player_url = 'http://player.ooyala.com/player.js?externalId=%s:%s&pcode=%s' % (partner_id, video_id, pcode) - return self._extract(player_url, video_id) + partner_id, video_id, pcode = re.match(self._VALID_URL, url).groups() + content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/external_id/%s/%s:%s' % (pcode, partner_id, video_id) + return self._extract(content_tree_url, video_id) From 497ca088a60fdd0a98f16e22a9d4fec135a26ab0 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 15 Oct 2015 14:37:05 +0100 Subject: [PATCH 009/110] [ooyala] remove print statment --- youtube_dl/extractor/ooyala.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index 592cdc564..df99a39f4 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -12,7 +12,6 @@ from ..utils import ( class OoyalaBaseIE(InfoExtractor): def _extract(self, player_url, video_id): - print(player_url) content_tree = self._download_json(player_url, video_id)['content_tree'] metadata = content_tree[list(content_tree)[0]] embed_code = metadata['embed_code'] From dd414c970bcc493358ff6a76f6544a0417125594 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 16 Oct 2015 10:12:42 +0100 Subject: [PATCH 010/110] [ooyala] fix sorting and format id --- youtube_dl/extractor/ooyala.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index df99a39f4..075b594ce 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -39,15 +39,15 @@ class OoyalaBaseIE(InfoExtractor): video_info['url'] = url return video_info if delivery_type == 'hls': - formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', 0, m3u8_id='hls', fatal=False)) + formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)) elif delivery_type == 'hds': - formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False)) + formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False)) else: formats.append({ 'url': url, 'ext': stream.get('delivery_type'), 'vcodec': stream.get('video_codec'), - 'format_id': stream.get('profile'), + 'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')), 'width': int_or_none(stream.get('width')), 'height': int_or_none(stream.get('height')), 'abr': int_or_none(stream.get('audio_bitrate')), From cce9d15d0115e8b4cd1f6e2a327b5e9dbdf0ee54 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 16 Oct 2015 16:02:40 +0100 Subject: [PATCH 011/110] [ooyala] extract domain,handle errors and change related tests --- youtube_dl/extractor/byutv.py | 5 ++- youtube_dl/extractor/generic.py | 9 ++-- youtube_dl/extractor/groupon.py | 2 + youtube_dl/extractor/howcast.py | 1 + youtube_dl/extractor/ooyala.py | 60 ++++++++++++++----------- youtube_dl/extractor/teachingchannel.py | 1 + youtube_dl/extractor/vice.py | 1 + 7 files changed, 48 insertions(+), 31 deletions(-) diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py index 3b2de517e..ce25816f0 100644 --- a/youtube_dl/extractor/byutv.py +++ b/youtube_dl/extractor/byutv.py @@ -14,9 +14,10 @@ class BYUtvIE(InfoExtractor): 'info_dict': { 'id': 'studio-c-season-5-episode-5', 'ext': 'mp4', - 'description': 'md5:5438d33774b6bdc662f9485a340401cc', + 'description': 'md5:e07269172baff037f8e8bf9956bc9747', 'title': 'Season 5 Episode 5', - 'thumbnail': 're:^https?://.*\.jpg$' + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 1486486, }, 'params': { 'skip_download': True, diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index ca5fbafb2..805677364 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -335,6 +335,7 @@ class GenericIE(InfoExtractor): 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ', 'ext': 'mp4', 'title': '2cc213299525360.mov', # that's what we get + 'duration': 238231, }, 'add_ie': ['Ooyala'], }, @@ -346,6 +347,7 @@ class GenericIE(InfoExtractor): 'ext': 'mp4', 'title': '"Steve Jobs: Man in the Machine" trailer', 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."', + 'duration': 135427, }, 'params': { 'skip_download': True, @@ -943,8 +945,9 @@ class GenericIE(InfoExtractor): 'info_dict': { 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs', 'ext': 'mp4', - 'description': 'VIDEO: Index/Match versus VLOOKUP.', + 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.', 'title': 'This is what separates the Excel masters from the wannabes', + 'duration': 191933, }, 'params': { # m3u8 downloads @@ -1454,7 +1457,7 @@ class GenericIE(InfoExtractor): re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P.{32})[\'"]\)', webpage) or re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P.{32})[\'"]', webpage)) if mobj is not None: - return OoyalaIE._build_url_result(mobj.group('ec')) + return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url})) # Look for multiple Ooyala embeds on SBN network websites mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage) @@ -1462,7 +1465,7 @@ class GenericIE(InfoExtractor): embeds = self._parse_json(mobj.group(1), video_id, fatal=False) if embeds: return _playlist_from_matches( - embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala') + embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala') # Look for Aparat videos mobj = re.search(r'