From 957e0db1d2d1d4b2e9fdd1c314b0c46e68ca0cb3 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 30 Oct 2015 13:56:21 +0100 Subject: [PATCH 001/182] [baidu] improve info extraction --- youtube_dl/extractor/baidu.py | 45 +++++++++++++---------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index e37ee4440..84fab551b 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -14,8 +14,8 @@ class BaiduVideoIE(InfoExtractor): 'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6', 'info_dict': { 'id': '1069', - 'title': '中华小当家 TV版 (全52集)', - 'description': 'md5:395a419e41215e531c857bb037bbaf80', + 'title': '中华小当家 TV版国语', + 'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0', }, 'playlist_count': 52, }, { @@ -25,45 +25,32 @@ class BaiduVideoIE(InfoExtractor): 'title': 're:^奔跑吧兄弟', 'description': 'md5:1bf88bad6d850930f542d51547c089b8', }, - 'playlist_mincount': 3, + 'playlist_mincount': 12, }] + def _call_api(self, path, category, playlist_id): + return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id) + def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') - category = category2 = mobj.group('type') + category, playlist_id = re.match(self._VALID_URL, url).groups() if category == 'show': - category2 = 'tvshow' + category = 'tvshow' + if category == 'tv': + category = 'tvplay' - webpage = self._download_webpage(url, playlist_id) + playlist_detail = self._call_api('xqinfo', category, playlist_id) - playlist_title = self._html_search_regex( - r'title\s*:\s*(["\'])(?P[^\']+)\1', webpage, - 'playlist title', group='title') - playlist_description = self._html_search_regex( - r'<input[^>]+class="j-data-intro"[^>]+value="([^"]+)"/>', webpage, - playlist_id, 'playlist description') + playlist_title = playlist_detail['title'] + playlist_description = playlist_detail.get('intro') - site = self._html_search_regex( - r'filterSite\s*:\s*["\']([^"]*)["\']', webpage, - 'primary provider site') - api_result = self._download_json( - 'http://v.baidu.com/%s_intro/?dtype=%sPlayUrl&id=%s&site=%s' % ( - category, category2, playlist_id, site), - playlist_id, 'Get playlist links') + episodes_detail = self._call_api('xqsingle', category, playlist_id) entries = [] - for episode in api_result[0]['episodes']: + for episode in episodes_detail['videos']: episode_id = '%s_%s' % (playlist_id, episode['episode']) - redirect_page = self._download_webpage( - compat_urlparse.urljoin(url, episode['url']), episode_id, - note='Download Baidu redirect page') - real_url = self._html_search_regex( - r'location\.replace\("([^"]+)"\)', redirect_page, 'real URL') - entries.append(self.url_result( - real_url, video_title=episode['single_title'])) + episode['url'], video_title=episode['title'])) return self.playlist_result( entries, playlist_id, playlist_title, playlist_description) From 5f432ac8f552b06e715c5e17165328dc76d9c1b5 Mon Sep 17 00:00:00 2001 From: Founder Fang <founder.fang@gmail.com> Date: Sun, 20 Dec 2015 19:09:45 +0800 Subject: [PATCH 002/182] [Weiqitv] Add new extractor --- youtube_dl/extractor/__init__.py | 4 ++- youtube_dl/extractor/letv.py | 57 ++++++++++++++++++++++++++++++++ youtube_dl/extractor/weiqitv.py | 54 ++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 youtube_dl/extractor/weiqitv.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index e46d73ed7..9dcd252f8 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -331,7 +331,8 @@ from .lecture2go import Lecture2GoIE from .letv import ( LetvIE, LetvTvIE, - LetvPlaylistIE + LetvPlaylistIE, + LetvCloudIE, ) from .libsyn import LibsynIE from .lifenews import ( @@ -834,6 +835,7 @@ from .webofstories import ( WebOfStoriesPlaylistIE, ) from .weibo import WeiboIE +from .weiqitv import WeiqitvIE from .wimp import WimpIE from .wistia import WistiaIE from .worldstarhiphop import WorldStarHipHopIE diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py index be648000e..c096cb1ab 100644 --- a/youtube_dl/extractor/letv.py +++ b/youtube_dl/extractor/letv.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import datetime import re import time +import base64 from .common import InfoExtractor from ..compat import ( @@ -16,6 +17,7 @@ from ..utils import ( parse_iso8601, sanitized_Request, int_or_none, + str_or_none, encode_data_uri, ) @@ -239,3 +241,58 @@ class LetvPlaylistIE(LetvTvIE): }, 'playlist_mincount': 7 }] + + +class LetvCloudIE(InfoExtractor): + IE_DESC = '乐视云' + _VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$' + + _TESTS = [{ + 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf', + 'md5': '26450599afd64c513bc77030ad15db44', + 'info_dict': { + 'id': 'p7jnfw5hw9_467623dedf', + 'ext': 'mp4', + 'title': 'p7jnfw5hw9_467623dedf', + }, + }, { + 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360', + 'info_dict': { + 'id': 'p7jnfw5hw9_ec93197892', + 'ext': 'mp4', + 'title': 'p7jnfw5hw9_ec93197892', + }, + }, { + 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd', + 'info_dict': { + 'id': 'p7jnfw5hw9_187060b6fd', + 'ext': 'mp4', + 'title': 'p7jnfw5hw9_187060b6fd', + }, + }] + + def _real_extract(self, url): + uu = re.search('uu=([\w]+)', url).group(1) + vu = re.search('vu=([\w]+)', url).group(1) + media_id = uu + '_' + vu + + play_json_req = sanitized_Request( + 'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' + + "uu=" + uu + "&vu=" + vu) + play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data') + + formats = [{ + 'url': base64.b64decode(media['play_url']['main_url'].encode('utf-8')).decode("utf-8"), + 'ext': 'mp4', + 'format_id': int_or_none(media.get('play_url', {}).get('vtype')), + 'format_note': str_or_none(media.get('play_url', {}).get('definition')), + 'width': int_or_none(media.get('play_url', {}).get('vwidth')), + 'height': int_or_none(media.get('play_url', {}).get('vheight')), + } for media in play_json['data']['video_info']['media'].values()] + self._sort_formats(formats) + + return { + 'id': media_id, + 'title': media_id, + 'formats': formats, + } diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py new file mode 100644 index 000000000..da3b3d145 --- /dev/null +++ b/youtube_dl/extractor/weiqitv.py @@ -0,0 +1,54 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class WeiqitvIE(InfoExtractor): + IE_DESC = 'WQTV' + _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)' + + _TESTS = [{ + 'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3', + 'md5': '26450599afd64c513bc77030ad15db44', + 'info_dict': { + 'id': '53c744f09874f0e76a8b46f3', + 'ext': 'mp4', + 'title': '2013年度盘点', + }, + }, { + 'url': 'http://www.weiqitv.com/index/video_play?videoId=567379a2d4c36cca518b4569', + 'info_dict': { + 'id': '567379a2d4c36cca518b4569', + 'ext': 'mp4', + 'title': '民国围棋史', + }, + }, { + 'url': 'http://www.weiqitv.com/index/video_play?videoId=5430220a9874f088658b4567', + 'info_dict': { + 'id': '5430220a9874f088658b4567', + 'ext': 'mp4', + 'title': '二路托过的手段和运用', + }, + }] + + def _real_extract(self, url): + media_id = self._match_id(url) + page = self._download_webpage(url, media_id) + + info_json_str = self._search_regex( + 'var\s+video\s*=\s*(.+});', + page, 'info_json_str') + info_json = self._parse_json(info_json_str, media_id) + + letvcloud_url = self._search_regex( + 'var\s+letvurl\s*=\s*"([^"]+)', + page, 'letvcloud_url') + + return { + '_type': 'url_transparent', + "ie_key": 'LetvCloud', + 'url': letvcloud_url, + 'title': info_json['name'], + 'id': media_id, + } From 7109903e6140e01b5d5980fc80c6ef1b6c86797a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 31 Dec 2015 03:10:44 +0600 Subject: [PATCH 003/182] [extractor/common] Document chapter and series fields --- youtube_dl/extractor/common.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 655207447..c443daf20 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -200,6 +200,26 @@ class InfoExtractor(object): end_time: Time in seconds where the reproduction should end, as specified in the URL. + The following fields should only be used when the video belongs to some logical + chapter or section: + + chapter: Name or title of the chapter the video belongs to. + chapter_id: Number or id of the chapter the video belongs to, as an integer + or unicode string. + + The following fields should only be used when the video is an episode of some + series or programme: + + series: Title of the series or programme the video episode belongs to. + season: Title of the season the video episode belongs to. + season_id: Number or id of the season the video episode belongs to, as an + integer or unicode string. + episode: Title of the video episode. Unlike mandatory video title field, + this field should denote the exact title of the video episode + without any kind of decoration. + episode_id: Number or id of the video episode within a season, as an integer + or unicode string. + Unless mentioned otherwise, the fields should be Unicode strings. Unless mentioned otherwise, None is equivalent to absence of information. From 4d402db52140458b1b4707c30fb01bb92861a8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 31 Dec 2015 03:11:21 +0600 Subject: [PATCH 004/182] [udemy] Extract chapter info --- youtube_dl/extractor/udemy.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index ce7e211fc..41097829d 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -244,10 +244,25 @@ class UdemyCourseIE(UdemyIE): 'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id, course_id, 'Downloading course curriculum') - entries = [ - self.url_result( - 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'Udemy') - for asset in response if asset.get('assetType') or asset.get('asset_type') == 'Video' - ] + entries = [] + chapter, chapter_id = None, None + for asset in response: + asset_type = asset.get('assetType') or asset.get('asset_type') + if asset_type == 'Video': + asset_id = asset.get('id') + if asset_id: + entry = { + '_type': 'url_transparent', + 'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), + 'ie_key': UdemyIE.ie_key(), + } + if chapter_id: + entry['chapter_id'] = chapter_id + if chapter: + entry['chapter'] = chapter + entries.append(entry) + elif asset.get('type') == 'chapter': + chapter_id = asset.get('index') or asset.get('object_index') + chapter = asset.get('title') return self.playlist_result(entries, course_id, course_title) From 7a0b07c719b621b1572120ed3f04870fa5737bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 31 Dec 2015 03:13:02 +0600 Subject: [PATCH 005/182] [videomore] Extract series info --- youtube_dl/extractor/videomore.py | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py index 16078ac19..3bd96e445 100644 --- a/youtube_dl/extractor/videomore.py +++ b/youtube_dl/extractor/videomore.py @@ -23,11 +23,56 @@ class VideomoreIE(InfoExtractor): 'ext': 'flv', 'title': 'В гостях Алексей Чумаков и Юлия Ковальчук', 'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.', + 'series': 'Кино в деталях', + 'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук', + 'episode_id': None, + 'season': 'Сезон 2015', + 'season_id': 5, 'thumbnail': 're:^https?://.*\.jpg', 'duration': 2910, 'age_limit': 16, 'view_count': int, }, + }, { + 'url': 'http://videomore.ru/embed/259974', + 'info_dict': { + 'id': '259974', + 'ext': 'flv', + 'title': '80 серия', + 'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.', + 'series': 'Молодежка', + 'episode': '80 серия', + 'episode_id': 40, + 'season': '2 сезон', + 'season_id': 2, + 'thumbnail': 're:^https?://.*\.jpg', + 'duration': 2809, + 'age_limit': 16, + 'view_count': int, + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'http://videomore.ru/molodezhka/sezon_promo/341073', + 'info_dict': { + 'id': '341073', + 'ext': 'flv', + 'title': 'Команда проиграла из-за Бакина?', + 'description': 'Молодежка 3 сезон скоро', + 'series': 'Молодежка', + 'episode': 'Команда проиграла из-за Бакина?', + 'episode_id': None, + 'season': 'Промо', + 'season_id': 99, + 'thumbnail': 're:^https?://.*\.jpg', + 'duration': 29, + 'age_limit': 16, + 'view_count': int, + }, + 'params': { + 'skip_download': True, + }, }, { 'url': 'http://videomore.ru/elki_3?track_id=364623', 'only_matching': True, @@ -81,10 +126,21 @@ class VideomoreIE(InfoExtractor): 'url': thumbnail, } for thumbnail in data.get('big_thumbnail_urls', [])] + series = data.get('project_title') + episode = data.get('title') + episode_id = data.get('episode_of_season') or None + season = data.get('season_title') + season_id = data.get('season_pos') or None + return { 'id': video_id, 'title': title, 'description': description, + 'series': series, + 'episode': episode, + 'episode_id': episode_id, + 'season': season, + 'season_id': season_id, 'thumbnails': thumbnails, 'timestamp': timestamp, 'duration': duration, From 0416006a3051b15e4bebbb096960ca4fb8ffd0a9 Mon Sep 17 00:00:00 2001 From: j <j@mailb.org> Date: Tue, 15 Dec 2015 21:37:47 +0100 Subject: [PATCH 006/182] Fix einthusan parser --- youtube_dl/extractor/einthusan.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py index 5dfea0d39..bc6def65e 100644 --- a/youtube_dl/extractor/einthusan.py +++ b/youtube_dl/extractor/einthusan.py @@ -41,9 +41,12 @@ class EinthusanIE(InfoExtractor): video_title = self._html_search_regex( r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title') - video_url = self._html_search_regex( - r'''(?s)jwplayer\("mediaplayer"\)\.setup\({.*?'file': '([^']+)'.*?}\);''', - webpage, 'video url') + movieid = self._html_search_regex( + r'data-movieid="(.*?)"', webpage, 'movieid') + + location = 'Washington' + geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location) + video_url = self._download_webpage(geturl, video_id) description = self._html_search_meta('description', webpage) thumbnail = self._html_search_regex( From b26afec81f408af37cca6298109ca2a59688bf79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 02:23:03 +0600 Subject: [PATCH 007/182] [einthusan] Improve extraction (Closes #7877) --- youtube_dl/extractor/einthusan.py | 34 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py index bc6def65e..f7339702c 100644 --- a/youtube_dl/extractor/einthusan.py +++ b/youtube_dl/extractor/einthusan.py @@ -1,9 +1,12 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor +from ..compat import compat_urlparse +from ..utils import ( + remove_start, + sanitized_Request, +) class EinthusanIE(InfoExtractor): @@ -34,30 +37,33 @@ class EinthusanIE(InfoExtractor): ] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - webpage = self._download_webpage(url, video_id) + video_id = self._match_id(url) - video_title = self._html_search_regex( - r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title') + request = sanitized_Request(url) + request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0') + webpage = self._download_webpage(request, video_id) - movieid = self._html_search_regex( - r'data-movieid="(.*?)"', webpage, 'movieid') + title = self._html_search_regex( + r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>', + webpage, 'title') - location = 'Washington' - geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location) - video_url = self._download_webpage(geturl, video_id) + video_id = self._search_regex( + r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id) + + video_url = self._download_webpage( + 'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/' + % video_id, video_id) description = self._html_search_meta('description', webpage) thumbnail = self._html_search_regex( r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''', webpage, "thumbnail url", fatal=False) if thumbnail is not None: - thumbnail = thumbnail.replace('..', 'http://www.einthusan.com') + thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..')) return { 'id': video_id, - 'title': video_title, + 'title': title, 'url': video_url, 'thumbnail': thumbnail, 'description': description, From c1e90619bde0c8b3b17d938195cfeb5ed7803125 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Thu, 31 Dec 2015 22:10:00 +0100 Subject: [PATCH 008/182] [mtv] extract mgid extraction and query building into separate methods --- youtube_dl/extractor/mtv.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index d887583e6..e8bb527b8 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -167,14 +167,16 @@ class MTVServicesInfoExtractor(InfoExtractor): 'description': description, } + def _get_feed_query(self, uri): + data = {'uri': uri} + if self._LANG: + data['lang'] = self._LANG + return compat_urllib_parse.urlencode(data) + def _get_videos_info(self, uri): video_id = self._id_from_uri(uri) feed_url = self._get_feed_url(uri) - data = compat_urllib_parse.urlencode({'uri': uri}) - info_url = feed_url + '?' - if self._LANG: - info_url += 'lang=%s&' % self._LANG - info_url += data + info_url = feed_url + '?' + self._get_feed_query(uri) return self._get_videos_info_from_url(info_url, video_id) def _get_videos_info_from_url(self, url, video_id): @@ -184,9 +186,7 @@ class MTVServicesInfoExtractor(InfoExtractor): return self.playlist_result( [self._get_video_info(item) for item in idoc.findall('.//item')]) - def _real_extract(self, url): - title = url_basename(url) - webpage = self._download_webpage(url, title) + def _extract_mgid(self, webpage): try: # the url can be http://media.mtvnservices.com/fb/{mgid}.swf # or http://media.mtvnservices.com/{mgid} @@ -207,7 +207,12 @@ class MTVServicesInfoExtractor(InfoExtractor): 'sm4:video:embed', webpage, 'sm4 embed', default='') mgid = self._search_regex( r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid') + return mgid + def _real_extract(self, url): + title = url_basename(url) + webpage = self._download_webpage(url, title) + mgid = self._extract_mgid(webpage) videos_info = self._get_videos_info(mgid) return videos_info From a0e5beb0fbfa5f300087c18b12ff67fade51d578 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Thu, 31 Dec 2015 22:12:05 +0100 Subject: [PATCH 009/182] [nick] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/nick.py | 63 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 youtube_dl/extractor/nick.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index e20d90bac..120ed9340 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -441,6 +441,7 @@ from .nhl import ( NHLNewsIE, NHLVideocenterIE, ) +from .nick import NickIE from .niconico import NiconicoIE, NiconicoPlaylistIE from .ninegag import NineGagIE from .noco import NocoIE diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py new file mode 100644 index 000000000..4840368ac --- /dev/null +++ b/youtube_dl/extractor/nick.py @@ -0,0 +1,63 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .mtv import MTVServicesInfoExtractor +from ..compat import compat_urllib_parse + + +class NickIE(MTVServicesInfoExtractor): + IE_NAME = 'nick.com' + _VALID_URL = r'https?://(?:www\.)?nick\.com/videos/clip/(?P<id>[^/?#.]+)' + _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm' + _TESTS = [{ + 'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html', + 'playlist': [ + { + 'md5': '6e5adc1e28253bbb1b28ab05403dd4d4', + 'info_dict': { + 'id': 'be6a17b0-412d-11e5-8ff7-0026b9414f30', + 'ext': 'mp4', + 'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S1', + 'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.', + + } + }, + { + 'md5': 'd7be441fc53a1d4882fa9508a1e5b3ce', + 'info_dict': { + 'id': 'be6b8f96-412d-11e5-8ff7-0026b9414f30', + 'ext': 'mp4', + 'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S2', + 'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.', + + } + }, + { + 'md5': 'efffe1728a234b2b0d2f2b343dd1946f', + 'info_dict': { + 'id': 'be6cf7e6-412d-11e5-8ff7-0026b9414f30', + 'ext': 'mp4', + 'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S3', + 'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.', + } + }, + { + 'md5': '1ec6690733ab9f41709e274a1d5c7556', + 'info_dict': { + 'id': 'be6e3354-412d-11e5-8ff7-0026b9414f30', + 'ext': 'mp4', + 'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S4', + 'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.', + } + }, + ], + }] + + def _get_feed_query(self, uri): + return compat_urllib_parse.urlencode({ + 'feed': 'nick_arc_player_prime', + 'mgid': uri + }) + + def _extract_mgid(self, webpage): + return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid') From e565cf6048739fdfdfbab19e4b35c37a53865807 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Thu, 31 Dec 2015 22:47:18 +0100 Subject: [PATCH 010/182] [nextmovie] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/nextmovie.py | 30 ++++++++++++++++++++++++++++++ youtube_dl/extractor/nick.py | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 youtube_dl/extractor/nextmovie.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 120ed9340..b3f7059e4 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -434,6 +434,7 @@ from .nextmedia import ( NextMediaActionNewsIE, AppleDailyIE, ) +from .nextmovie import NextMovieIE from .nfb import NFBIE from .nfl import NFLIE from .nhl import ( diff --git a/youtube_dl/extractor/nextmovie.py b/youtube_dl/extractor/nextmovie.py new file mode 100644 index 000000000..657ae77a0 --- /dev/null +++ b/youtube_dl/extractor/nextmovie.py @@ -0,0 +1,30 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .mtv import MTVServicesInfoExtractor +from ..compat import compat_urllib_parse + + +class NextMovieIE(MTVServicesInfoExtractor): + IE_NAME = 'nextmovie.com' + _VALID_URL = r'https?://(?:www\.)?nextmovie\.com/shows/[^/]+/\d{4}-\d{2}-\d{2}/(?P<id>[^/?#]+)' + _FEED_URL = 'http://lite.dextr.mtvi.com/service1/dispatch.htm' + _TESTS = [{ + 'url': 'http://www.nextmovie.com/shows/exclusives/2013-03-10/mgid:uma:videolist:nextmovie.com:1715019/', + 'md5': '09a9199f2f11f10107d04fcb153218aa', + 'info_dict': { + 'id': '961726', + 'ext': 'mp4', + 'title': 'The Muppets\' Gravity', + }, + }] + + def _get_feed_query(self, uri): + return compat_urllib_parse.urlencode({ + 'feed': '1505', + 'mgid': uri, + }) + + def _real_extract(self, url): + mgid = self._match_id(url) + return self._get_videos_info(mgid) diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py index 4840368ac..b62819ae5 100644 --- a/youtube_dl/extractor/nick.py +++ b/youtube_dl/extractor/nick.py @@ -56,7 +56,7 @@ class NickIE(MTVServicesInfoExtractor): def _get_feed_query(self, uri): return compat_urllib_parse.urlencode({ 'feed': 'nick_arc_player_prime', - 'mgid': uri + 'mgid': uri, }) def _extract_mgid(self, webpage): From 034caf70b24da1d1de12b00e9ac5620fb7664220 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Thu, 31 Dec 2015 13:05:46 +0800 Subject: [PATCH 011/182] [youku] Fix extraction (#8068) --- youtube_dl/extractor/youku.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py index 3a3432be8..f767fa15f 100644 --- a/youtube_dl/extractor/youku.py +++ b/youtube_dl/extractor/youku.py @@ -2,6 +2,9 @@ from __future__ import unicode_literals import base64 +import random +import string +import time from .common import InfoExtractor from ..compat import ( @@ -141,6 +144,11 @@ class YoukuIE(InfoExtractor): return video_urls_dict + @staticmethod + def get_ysuid(): + return '%d%s' % (int(time.time()), ''.join([ + random.choice(string.ascii_letters) for i in range(3)])) + def get_hd(self, fm): hd_id_dict = { '3gp': '0', @@ -189,6 +197,8 @@ class YoukuIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) + self._set_cookie('youku.com', '__ysuid', self.get_ysuid()) + def retrieve_data(req_url, note): headers = { 'Referer': req_url, From 0d5095fc65d92251f7aeefd9cb504434590cee8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 15:14:41 +0600 Subject: [PATCH 012/182] [ccc] Update _VALID_URL (Closes #8097) --- youtube_dl/extractor/ccc.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 6924eac70..2e0a0b29f 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -12,13 +12,13 @@ from ..utils import ( class CCCIE(InfoExtractor): IE_NAME = 'media.ccc.de' - _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html' + _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/v/(?P<id>[^/?#&]+)' - _TEST = { - 'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video', + _TESTS = [{ + 'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video', 'md5': '3a1eda8f3a29515d27f5adb967d7e740', 'info_dict': { - 'id': '20131228183', + 'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor', 'ext': 'mp4', 'title': 'Introduction to Processor Design', 'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b', @@ -26,7 +26,10 @@ class CCCIE(InfoExtractor): 'view_count': int, 'upload_date': '20131229', } - } + }, { + 'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download', + 'only_matching': True, + }] def _real_extract(self, url): video_id = self._match_id(url) From c9154514c495bd2259df2250d4cf72a2564a9136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 15:22:22 +0600 Subject: [PATCH 013/182] [ccc] Fix upload date extraction --- youtube_dl/extractor/ccc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 2e0a0b29f..5b549b343 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -46,7 +46,7 @@ class CCCIE(InfoExtractor): r"(?s)<p class='description'>(.*?)</p>", webpage, 'description', fatal=False) upload_date = unified_strdate(self._html_search_regex( - r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>", + r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>", webpage, 'upload date', fatal=False)) view_count = int_or_none(self._html_search_regex( r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>", From 8499d211583f3534c0d0e0f086d471c52ac6c803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 15:29:42 +0600 Subject: [PATCH 014/182] [ccc] Fix description extraction and update test --- youtube_dl/extractor/ccc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 5b549b343..45115c838 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -21,10 +21,10 @@ class CCCIE(InfoExtractor): 'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor', 'ext': 'mp4', 'title': 'Introduction to Processor Design', - 'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b', + 'description': 'md5:80be298773966f66d56cb11260b879af', 'thumbnail': 're:^https?://.*\.jpg$', 'view_count': int, - 'upload_date': '20131229', + 'upload_date': '20131228', } }, { 'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download', @@ -43,7 +43,7 @@ class CCCIE(InfoExtractor): title = self._html_search_regex( r'(?s)<h1>(.*?)</h1>', webpage, 'title') description = self._html_search_regex( - r"(?s)<p class='description'>(.*?)</p>", + r"(?s)<h3>About</h3>(.+?)<h3>", webpage, 'description', fatal=False) upload_date = unified_strdate(self._html_search_regex( r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>", From 82597f0ec0b8ec208e77d27f62904de7804b914d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 15:41:52 +0600 Subject: [PATCH 015/182] [ccc] Extract duration --- youtube_dl/extractor/ccc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 45115c838..710e5919c 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -5,6 +5,7 @@ import re from .common import InfoExtractor from ..utils import ( int_or_none, + parse_duration, qualities, unified_strdate, ) @@ -25,6 +26,7 @@ class CCCIE(InfoExtractor): 'thumbnail': 're:^https?://.*\.jpg$', 'view_count': int, 'upload_date': '20131228', + 'duration': 3660, } }, { 'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download', @@ -51,6 +53,9 @@ class CCCIE(InfoExtractor): view_count = int_or_none(self._html_search_regex( r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>", webpage, 'view count', fatal=False)) + duration = parse_duration(self._html_search_regex( + r'(?s)<span[^>]+class=(["\']).*?fa-clock-o.*?\1[^>]*></span>(?P<duration>.+?)</li', + webpage, 'duration', fatal=False, group='duration')) matches = re.finditer(r'''(?xs) <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s* @@ -98,5 +103,6 @@ class CCCIE(InfoExtractor): 'thumbnail': thumbnail, 'view_count': view_count, 'upload_date': upload_date, + 'duration': duration, 'formats': formats, } From 190ef0798103908d986e07f54323299084726d42 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 1 Jan 2016 12:17:10 +0100 Subject: [PATCH 016/182] release 2016.01.01 --- docs/supportedsites.md | 7 +++++-- youtube_dl/version.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 2297edc6b..84c166805 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -28,7 +28,6 @@ - **AlJazeera** - **Allocine** - **AlphaPorno** - - **AnimalPlanet** - **anitube.se** - **AnySex** - **Aparat** @@ -368,11 +367,13 @@ - **Newstube** - **NextMedia**: 蘋果日報 - **NextMediaActionNews**: 蘋果日報 - 動新聞 + - **nextmovie.com** - **nfb**: National Film Board of Canada - **nfl.com** - **nhl.com** - **nhl.com:news**: NHL news - **nhl.com:videocenter**: NHL videocenter category + - **nick.com** - **niconico**: ニコニコ動画 - **NiconicoPlaylist** - **njoy**: N-JOY @@ -411,6 +412,7 @@ - **orf:iptv**: iptv.ORF.at - **orf:oe1**: Radio Österreich 1 - **orf:tvthek**: ORF TVthek + - **pandora.tv**: 판도라TV - **parliamentlive.tv**: UK parliament videos - **Patreon** - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC) @@ -459,6 +461,7 @@ - **RBMARadio** - **RDS**: RDS.ca - **RedTube** + - **RegioTV** - **Restudy** - **ReverbNation** - **RingTV** @@ -582,7 +585,6 @@ - **THVideo** - **THVideoPlaylist** - **tinypic**: tinypic.com videos - - **tlc.com** - **tlc.de** - **TMZ** - **TMZArticle** @@ -611,6 +613,7 @@ - **TVC** - **TVCArticle** - **tvigle**: Интернет-телевидение Tvigle.ru + - **tvland.com** - **tvp.pl** - **tvp.pl:Series** - **TVPlay**: TV3Play and related services diff --git a/youtube_dl/version.py b/youtube_dl/version.py index cf6c4d74d..790bd5b3b 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2015.12.31' +__version__ = '2016.01.01' From 32f9036447d1211f9ce0750203d71671f0ee99dc Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 1 Jan 2016 13:28:45 +0100 Subject: [PATCH 017/182] [ccc] Add language information to formats --- youtube_dl/YoutubeDL.py | 4 ++++ youtube_dl/extractor/ccc.py | 8 ++++++-- youtube_dl/extractor/common.py | 5 +++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 50425b8d7..3b2be3159 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1791,6 +1791,10 @@ class YoutubeDL(object): res = '' if fdict.get('ext') in ['f4f', 'f4m']: res += '(unsupported) ' + if fdict.get('language'): + if res: + res += ' ' + res += '[%s]' % fdict['language'] if fdict.get('format_note') is not None: res += fdict['format_note'] + ' ' if fdict.get('tbr') is not None: diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 710e5919c..e94b1e35b 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -58,11 +58,12 @@ class CCCIE(InfoExtractor): webpage, 'duration', fatal=False, group='duration')) matches = re.finditer(r'''(?xs) - <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s* + <(?:span|div)\s+class='label\s+filetype'>(?P<format>[^<]*)</(?:span|div)>\s* + <(?:span|div)\s+class='label\s+filetype'>(?P<lang>[^<]*)</(?:span|div)>\s* <a\s+download\s+href='(?P<http_url>[^']+)'>\s* (?: .*? - <a\s+href='(?P<torrent_url>[^']+\.torrent)' + <a\s+(?:download\s+)?href='(?P<torrent_url>[^']+\.torrent)' )?''', webpage) formats = [] for m in matches: @@ -70,12 +71,15 @@ class CCCIE(InfoExtractor): format_id = self._search_regex( r'.*/([a-z0-9_-]+)/[^/]*$', m.group('http_url'), 'format id', default=None) + if format_id: + format_id = m.group('lang') + '-' + format_id vcodec = 'h264' if 'h264' in format_id else ( 'none' if format_id in ('mp3', 'opus') else None ) formats.append({ 'format_id': format_id, 'format': format, + 'language': m.group('lang'), 'url': m.group('http_url'), 'vcodec': vcodec, 'preference': preference(format_id), diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 655207447..2823b1d18 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -108,8 +108,9 @@ class InfoExtractor(object): -2 or smaller for less than default. < -1000 to hide the format (if there is another one which is strictly better) - * language_preference Is this in the correct requested - language? + * language Language code, e.g. "de" or "en-US". + * language_preference Is this in the language mentioned in + the URL? 10 if it's what the URL is about, -1 for default (don't know), -10 otherwise, other values reserved for now. From ca227c8698340ad9170698cef81ab4bf4d832a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 1 Jan 2016 14:32:00 +0100 Subject: [PATCH 018/182] [yahoo] Support pages that use an alias (fixes #8084) --- youtube_dl/extractor/yahoo.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index fca5ddc69..4a492f784 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -155,7 +155,16 @@ class YahooIE(InfoExtractor): 'description': 'md5:8fc39608213295748e1e289807838c97', 'duration': 1646, }, - } + }, { + # it uses an alias to get the video_id + 'url': 'https://www.yahoo.com/movies/the-stars-of-daddys-home-have-very-different-212843197.html', + 'info_dict': { + 'id': '40eda9c8-8e5f-3552-8745-830f67d0c737', + 'ext': 'mp4', + 'title': 'Will Ferrell & Mark Wahlberg Are Pro-Spanking', + 'description': 'While they play feuding fathers in \'Daddy\'s Home,\' star Will Ferrell & Mark Wahlberg share their true feelings on parenthood.', + }, + }, ] def _real_extract(self, url): @@ -199,13 +208,22 @@ class YahooIE(InfoExtractor): r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE, default=None) if items_json is None: - CONTENT_ID_REGEXES = [ - r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"', - r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"', - r'"first_videoid"\s*:\s*"([^"]+)"', - r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id), - ] - video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID') + alias = self._search_regex( + r'"aliases":{"video":"(.*?)"', webpage, 'alias', default=None) + if alias is not None: + alias_info = self._download_json( + 'https://www.yahoo.com/_td/api/resource/VideoService.videos;video_aliases=["%s"]' % alias, + display_id, 'Downloading alias info') + video_id = alias_info[0]['id'] + else: + CONTENT_ID_REGEXES = [ + r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"', + r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"', + r'"first_videoid"\s*:\s*"([^"]+)"', + r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id), + ] + video_id = self._search_regex( + CONTENT_ID_REGEXES, webpage, 'content ID') else: items = json.loads(items_json) info = items['mediaItems']['query']['results']['mediaObj'][0] From 27bfd4e5266623d9000ba17e6bb2a72eefb0e22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 20:26:56 +0600 Subject: [PATCH 019/182] [extractor/common] Introduce number fields for chapters and series --- youtube_dl/extractor/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index c443daf20..c63157619 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -204,21 +204,21 @@ class InfoExtractor(object): chapter or section: chapter: Name or title of the chapter the video belongs to. - chapter_id: Number or id of the chapter the video belongs to, as an integer - or unicode string. + chapter_number: Number of the chapter the video belongs to, as an integer. + chapter_id: Id of the chapter the video belongs to, as a unicode string. The following fields should only be used when the video is an episode of some series or programme: series: Title of the series or programme the video episode belongs to. season: Title of the season the video episode belongs to. - season_id: Number or id of the season the video episode belongs to, as an - integer or unicode string. + season_number: Number of the season the video episode belongs to, as an integer. + season_id: Id of the season the video episode belongs to, as a unicode string. episode: Title of the video episode. Unlike mandatory video title field, this field should denote the exact title of the video episode without any kind of decoration. - episode_id: Number or id of the video episode within a season, as an integer - or unicode string. + episode_number: Number of the video episode within a season, as an integer. + episode_id: Id of the video episode, as a unicode string. Unless mentioned otherwise, the fields should be Unicode strings. From 306c51c66922abb1846640b81c9b7c4d6a570d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 20:30:08 +0600 Subject: [PATCH 020/182] [videomore] Use number fields for series --- youtube_dl/extractor/videomore.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py index 3bd96e445..a66d6de23 100644 --- a/youtube_dl/extractor/videomore.py +++ b/youtube_dl/extractor/videomore.py @@ -25,9 +25,9 @@ class VideomoreIE(InfoExtractor): 'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.', 'series': 'Кино в деталях', 'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук', - 'episode_id': None, + 'episode_number': None, 'season': 'Сезон 2015', - 'season_id': 5, + 'season_number': 5, 'thumbnail': 're:^https?://.*\.jpg', 'duration': 2910, 'age_limit': 16, @@ -42,9 +42,9 @@ class VideomoreIE(InfoExtractor): 'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.', 'series': 'Молодежка', 'episode': '80 серия', - 'episode_id': 40, + 'episode_number': 40, 'season': '2 сезон', - 'season_id': 2, + 'season_number': 2, 'thumbnail': 're:^https?://.*\.jpg', 'duration': 2809, 'age_limit': 16, @@ -62,9 +62,9 @@ class VideomoreIE(InfoExtractor): 'description': 'Молодежка 3 сезон скоро', 'series': 'Молодежка', 'episode': 'Команда проиграла из-за Бакина?', - 'episode_id': None, + 'episode_number': None, 'season': 'Промо', - 'season_id': 99, + 'season_number': 99, 'thumbnail': 're:^https?://.*\.jpg', 'duration': 29, 'age_limit': 16, @@ -128,9 +128,9 @@ class VideomoreIE(InfoExtractor): series = data.get('project_title') episode = data.get('title') - episode_id = data.get('episode_of_season') or None + episode_number = int_or_none(data.get('episode_of_season') or None) season = data.get('season_title') - season_id = data.get('season_pos') or None + season_number = int_or_none(data.get('season_pos') or None) return { 'id': video_id, @@ -138,9 +138,9 @@ class VideomoreIE(InfoExtractor): 'description': description, 'series': series, 'episode': episode, - 'episode_id': episode_id, + 'episode_number': episode_number, 'season': season, - 'season_id': season_id, + 'season_number': season_number, 'thumbnails': thumbnails, 'timestamp': timestamp, 'duration': duration, From 5bafcf65255261c2dc6482de6b1e545e5de1cbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 1 Jan 2016 20:34:29 +0600 Subject: [PATCH 021/182] [udemy] Use chapter_number --- youtube_dl/extractor/udemy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 41097829d..1df636779 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -245,7 +245,7 @@ class UdemyCourseIE(UdemyIE): course_id, 'Downloading course curriculum') entries = [] - chapter, chapter_id = None, None + chapter, chapter_number = None, None for asset in response: asset_type = asset.get('assetType') or asset.get('asset_type') if asset_type == 'Video': @@ -256,13 +256,13 @@ class UdemyCourseIE(UdemyIE): 'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'ie_key': UdemyIE.ie_key(), } - if chapter_id: - entry['chapter_id'] = chapter_id + if chapter_number: + entry['chapter_number'] = chapter_number if chapter: entry['chapter'] = chapter entries.append(entry) elif asset.get('type') == 'chapter': - chapter_id = asset.get('index') or asset.get('object_index') + chapter_number = asset.get('index') or asset.get('object_index') chapter = asset.get('title') return self.playlist_result(entries, course_id, course_title) From 054479754c661f40140f5db7f58916a0006c6b10 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Fri, 1 Jan 2016 21:03:16 +0100 Subject: [PATCH 022/182] [revision3] Add new extractor(closes #6388) - revision3.com - testtube.com - animalist.com --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/revision3.py | 103 ++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 youtube_dl/extractor/revision3.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index b3f7059e4..7adce5499 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -557,6 +557,7 @@ from .redtube import RedTubeIE from .regiotv import RegioTVIE from .restudy import RestudyIE from .reverbnation import ReverbNationIE +from .revision3 import Revision3IE from .ringtv import RingTVIE from .ro220 import Ro220IE from .rottentomatoes import RottenTomatoesIE diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py new file mode 100644 index 000000000..25fe4ef16 --- /dev/null +++ b/youtube_dl/extractor/revision3.py @@ -0,0 +1,103 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_str +from ..utils import ( + int_or_none, + parse_iso8601, + unescapeHTML, +) + + +class Revision3IE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:revision3|testtube|animalist)\.com)/(?P<id>[^/]+(?:/[^/?#]+)?)' + _TESTS = [{ + 'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016', + 'md5': 'd94a72d85d0a829766de4deb8daaf7df', + 'info_dict': { + 'id': '73034', + 'ext': 'webm', + 'title': '5 Google Predictions for 2016', + 'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.', + 'upload_date': '20151228', + 'timestamp': 1451325600, + 'duration': 187, + } + }, { + 'url': 'http://testtube.com/brainstuff', + 'info_dict': { + 'id': '251', + 'title': 'BrainStuff', + 'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.', + }, + 'playlist_mincount': 93, + }] + _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s' + _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62' + + def _real_extract(self, url): + domain, display_id = re.match(self._VALID_URL, url).groups() + page_info = self._download_json( + self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id) + + if page_info['data']['type'] == 'episode': + episode_data = page_info['data'] + video_id = compat_str(episode_data['video']['data']['id']) + video_data = self._download_json( + 'http://revision3.com/api/getPlaylist.json?api_key=%s&codecs=h264,vp8,theora&video_id=%s' % (self._API_KEY, video_id), + video_id)['items'][0] + + formats = [] + for media_type, media in video_data['media'].items(): + for quality_id, quality in media.items(): + if quality_id == 'hls': + formats.extend(self._extract_m3u8_formats( + quality['url'], video_id, 'mp4', + 'm3u8_native', m3u8_id='hls', fatal=False)) + else: + formats.append({ + 'url': quality['url'], + 'format_id': '%s-%s' % (media_type, quality_id), + 'tbr': int_or_none(quality.get('bitrate')), + }) + self._sort_formats(formats) + + thumbnails = [{ + 'url': image_url, + 'id': image_id, + } for image_id, image_url in video_data.get('images', {}).items()] + + return { + 'id': video_id, + 'title': unescapeHTML(video_data['title']), + 'description': unescapeHTML(video_data.get('summary')), + 'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '), + 'author': episode_data.get('author'), + 'duration': int_or_none(video_data.get('duration')), + 'thumbnails': thumbnails, + 'formats': formats, + } + else: + show_data = page_info['show']['data'] + episodes_data = page_info['episodes']['data'] + num_episodes = page_info['meta']['totalEpisodes'] + processed_episodes = 0 + entries = [] + page_num = 1 + while True: + entries.extend([self.url_result( + url + '/%s' % episode['slug']) for episode in episodes_data]) + processed_episodes += len(episodes_data) + if processed_episodes == num_episodes: + break + page_num += 1 + episodes_data = self._download_json(self._PAGE_DATA_TEMPLATE % ( + domain, display_id + '/' + compat_str(page_num), domain), + display_id)['episodes']['data'] + + return self.playlist_result( + entries, compat_str(show_data['id']), + show_data.get('name'), show_data.get('summary')) From 8af2804a5d09e08d8916e7a7d2eab40864e9ecd8 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Fri, 1 Jan 2016 21:53:19 +0100 Subject: [PATCH 023/182] [testtube] Remove Extractor --- youtube_dl/extractor/revision3.py | 30 +++++++++-- youtube_dl/extractor/testtube.py | 90 ------------------------------- 2 files changed, 27 insertions(+), 93 deletions(-) delete mode 100644 youtube_dl/extractor/testtube.py diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py index 25fe4ef16..b1b8800b9 100644 --- a/youtube_dl/extractor/revision3.py +++ b/youtube_dl/extractor/revision3.py @@ -9,6 +9,7 @@ from ..utils import ( int_or_none, parse_iso8601, unescapeHTML, + qualities, ) @@ -19,12 +20,15 @@ class Revision3IE(InfoExtractor): 'md5': 'd94a72d85d0a829766de4deb8daaf7df', 'info_dict': { 'id': '73034', + 'display_id': 'technobuffalo/5-google-predictions-for-2016', 'ext': 'webm', 'title': '5 Google Predictions for 2016', 'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.', 'upload_date': '20151228', 'timestamp': 1451325600, 'duration': 187, + 'uploader': 'TechnoBuffalo', + 'uploader_id': 'technobuffalo', } }, { 'url': 'http://testtube.com/brainstuff', @@ -34,6 +38,20 @@ class Revision3IE(InfoExtractor): 'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.', }, 'playlist_mincount': 93, + }, { + 'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial', + 'info_dict': { + 'id': '60163', + 'display_id': 'dnews/5-weird-ways-plants-can-eat-animals', + 'duration': 275, + 'ext': 'webm', + 'title': '5 Weird Ways Plants Can Eat Animals', + 'description': 'Why have some plants evolved to eat meat?', + 'upload_date': '20150120', + 'timestamp': 1421763300, + 'uploader': 'DNews', + 'uploader_id': 'dnews', + }, }] _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s' _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62' @@ -51,7 +69,7 @@ class Revision3IE(InfoExtractor): video_id)['items'][0] formats = [] - for media_type, media in video_data['media'].items(): + for vcodec, media in video_data['media'].items(): for quality_id, quality in media.items(): if quality_id == 'hls': formats.extend(self._extract_m3u8_formats( @@ -60,22 +78,28 @@ class Revision3IE(InfoExtractor): else: formats.append({ 'url': quality['url'], - 'format_id': '%s-%s' % (media_type, quality_id), + 'format_id': '%s-%s' % (vcodec, quality_id), 'tbr': int_or_none(quality.get('bitrate')), + 'vcodec': vcodec, }) self._sort_formats(formats) + preference = qualities(['mini', 'small', 'medium', 'large']) thumbnails = [{ 'url': image_url, 'id': image_id, + 'preference': preference(image_id) } for image_id, image_url in video_data.get('images', {}).items()] return { 'id': video_id, + 'display_id': display_id, 'title': unescapeHTML(video_data['title']), 'description': unescapeHTML(video_data.get('summary')), 'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '), 'author': episode_data.get('author'), + 'uploader': video_data.get('show', {}).get('name'), + 'uploader_id': video_data.get('show', {}).get('slug'), 'duration': int_or_none(video_data.get('duration')), 'thumbnails': thumbnails, 'formats': formats, @@ -89,7 +113,7 @@ class Revision3IE(InfoExtractor): page_num = 1 while True: entries.extend([self.url_result( - url + '/%s' % episode['slug']) for episode in episodes_data]) + 'http://%s/%s/%s' % (domain, display_id, episode['slug'])) for episode in episodes_data]) processed_episodes += len(episodes_data) if processed_episodes == num_episodes: break diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py deleted file mode 100644 index 26655d690..000000000 --- a/youtube_dl/extractor/testtube.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..utils import ( - int_or_none, - qualities, -) - - -class TestTubeIE(InfoExtractor): - _VALID_URL = r'https?://testtube\.com/[^/?#]+/(?P<id>[^/?#]+)' - _TESTS = [{ - 'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial', - 'info_dict': { - 'id': '60163', - 'display_id': '5-weird-ways-plants-can-eat-animals', - 'duration': 275, - 'ext': 'webm', - 'title': '5 Weird Ways Plants Can Eat Animals', - 'description': 'Why have some plants evolved to eat meat?', - 'thumbnail': 're:^https?://.*\.jpg$', - 'uploader': 'DNews', - 'uploader_id': 'dnews', - }, - }, { - 'url': 'https://testtube.com/iflscience/insane-jet-ski-flipping', - 'info_dict': { - 'id': 'fAGfJ4YjVus', - 'ext': 'mp4', - 'title': 'Flipping Jet-Ski Skills | Outrageous Acts of Science', - 'uploader': 'Science Channel', - 'uploader_id': 'ScienceChannel', - 'upload_date': '20150203', - 'description': 'md5:e61374030015bae1d2e22f096d4769d6', - } - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage(url, display_id) - - youtube_url = self._html_search_regex( - r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"', - webpage, 'youtube iframe', default=None) - if youtube_url: - return self.url_result(youtube_url, 'Youtube', video_id=display_id) - - video_id = self._search_regex( - r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);", - webpage, 'video ID') - - all_info = self._download_json( - 'https://testtube.com/api/getPlaylist.json?api_key=ba9c741bce1b9d8e3defcc22193f3651b8867e62&codecs=h264,vp8,theora&video_id=%s' % video_id, - video_id) - info = all_info['items'][0] - - formats = [] - for vcodec, fdatas in info['media'].items(): - for name, fdata in fdatas.items(): - formats.append({ - 'format_id': '%s-%s' % (vcodec, name), - 'url': fdata['url'], - 'vcodec': vcodec, - 'tbr': fdata.get('bitrate'), - }) - self._sort_formats(formats) - - duration = int_or_none(info.get('duration')) - images = info.get('images') - thumbnails = None - preference = qualities(['mini', 'small', 'medium', 'large']) - if images: - thumbnails = [{ - 'id': thumbnail_id, - 'url': img_url, - 'preference': preference(thumbnail_id) - } for thumbnail_id, img_url in images.items()] - - return { - 'id': video_id, - 'display_id': display_id, - 'title': info['title'], - 'description': info.get('summary'), - 'thumbnails': thumbnails, - 'uploader': info.get('show', {}).get('name'), - 'uploader_id': info.get('show', {}).get('slug'), - 'duration': duration, - 'formats': formats, - } From 94de6cf59cb1933e395775f9ffca4fa311adb0dc Mon Sep 17 00:00:00 2001 From: pingtux <pingtux@users.noreply.github.com> Date: Sat, 2 Jan 2016 01:35:09 +0100 Subject: [PATCH 024/182] Remove testtube import Extractor got deleted in remitamine/youtube-dl@8af2804 --- youtube_dl/extractor/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 7adce5499..4c7e5223d 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -687,7 +687,6 @@ from .telemb import TeleMBIE from .teletask import TeleTaskIE from .tenplay import TenPlayIE from .testurl import TestURLIE -from .testtube import TestTubeIE from .tf1 import TF1IE from .theintercept import TheInterceptIE from .theonion import TheOnionIE From 76a353c9e5511a522b5331cb3f74d4215341791c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 2 Jan 2016 07:44:30 +0600 Subject: [PATCH 025/182] [ruutu] Fix extraction (Closes #8107) --- youtube_dl/extractor/ruutu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py index 41fddc375..ffea438cc 100644 --- a/youtube_dl/extractor/ruutu.py +++ b/youtube_dl/extractor/ruutu.py @@ -75,9 +75,12 @@ class RuutuIE(InfoExtractor): preference = -1 if proto == 'rtmp' else 1 label = child.get('label') tbr = int_or_none(child.get('bitrate')) + format_id = '%s-%s' % (proto, label if label else tbr) if label or tbr else proto + if not self._is_valid_url(video_url, video_id, format_id): + continue width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]] formats.append({ - 'format_id': '%s-%s' % (proto, label if label else tbr), + 'format_id': format_id, 'url': video_url, 'width': width, 'height': height, From f20a11ed257f21a1c658f827e0c2129a72582adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 2 Jan 2016 19:22:39 +0600 Subject: [PATCH 026/182] [bbccouk] Extend _VALID_URL (Closes #8116) --- youtube_dl/extractor/bbc.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py index 923273fb2..5c621a32b 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dl/extractor/bbc.py @@ -23,7 +23,17 @@ class BBCCoUkIE(InfoExtractor): IE_NAME = 'bbc.co.uk' IE_DESC = 'BBC iPlayer' _ID_REGEX = r'[pb][\da-z]{7}' - _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>%s)' % _ID_REGEX + _VALID_URL = r'''(?x) + https?:// + (?:www\.)?bbc\.co\.uk/ + (?: + programmes/(?!articles/)| + iplayer(?:/[^/]+)?/(?:episode/|playlist/)| + music/clips[/#]| + radio/player/ + ) + (?P<id>%s) + ''' % _ID_REGEX _MEDIASELECTOR_URLS = [ # Provides HQ HLS streams with even better quality that pc mediaset but fails @@ -193,6 +203,9 @@ class BBCCoUkIE(InfoExtractor): }, { 'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo', 'only_matching': True, + }, { + 'url': 'http://www.bbc.co.uk/radio/player/p03cchwf', + 'only_matching': True, } ] From c579c5e967cd223baa2c1f593b6ed65ac8643e42 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:31:02 +0800 Subject: [PATCH 027/182] [baidu] Cleanups --- youtube_dl/extractor/baidu.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index 84fab551b..734b50d1d 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_urlparse class BaiduVideoIE(InfoExtractor): @@ -47,8 +46,6 @@ class BaiduVideoIE(InfoExtractor): entries = [] for episode in episodes_detail['videos']: - episode_id = '%s_%s' % (playlist_id, episode['episode']) - entries.append(self.url_result( episode['url'], video_title=episode['title'])) From a1d9f6c5dc15f1d62e0ab8c9dd6feaed4ff83a34 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:36:35 +0800 Subject: [PATCH 028/182] [baidu] Improve playlist description --- youtube_dl/extractor/baidu.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index 734b50d1d..d60cb0fdd 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..utils import unescapeHTML class BaiduVideoIE(InfoExtractor): @@ -14,7 +15,7 @@ class BaiduVideoIE(InfoExtractor): 'info_dict': { 'id': '1069', 'title': '中华小当家 TV版国语', - 'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0', + 'description': 'md5:51be07afe461cf99fa61231421b5397c', }, 'playlist_count': 52, }, { @@ -40,7 +41,7 @@ class BaiduVideoIE(InfoExtractor): playlist_detail = self._call_api('xqinfo', category, playlist_id) playlist_title = playlist_detail['title'] - playlist_description = playlist_detail.get('intro') + playlist_description = unescapeHTML(playlist_detail.get('intro')) episodes_detail = self._call_api('xqsingle', category, playlist_id) From 88fb59d91bf5fd33dbe7863fbf1e4a9a641fa483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 2 Jan 2016 19:42:11 +0600 Subject: [PATCH 029/182] [bbccouk] Extend title extraction --- youtube_dl/extractor/bbc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py index 5c621a32b..7b169881a 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dl/extractor/bbc.py @@ -482,7 +482,8 @@ class BBCCoUkIE(InfoExtractor): if programme_id: formats, subtitles = self._download_media_selector(programme_id) - title = self._og_search_title(webpage) + title = self._og_search_title(webpage, default=None) or self._html_search_regex( + r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>', webpage, 'title') description = self._search_regex( r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>', webpage, 'description', default=None) From 03116772583103ba97eaf8ce2cbabba9742e1929 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:44:49 +0800 Subject: [PATCH 030/182] [baidu] Add notes for API calls --- youtube_dl/extractor/baidu.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index d60cb0fdd..d35116a7c 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -28,8 +28,9 @@ class BaiduVideoIE(InfoExtractor): 'playlist_mincount': 12, }] - def _call_api(self, path, category, playlist_id): - return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id) + def _call_api(self, path, category, playlist_id, note): + return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % ( + path, category, playlist_id), playlist_id, note) def _real_extract(self, url): category, playlist_id = re.match(self._VALID_URL, url).groups() @@ -38,12 +39,14 @@ class BaiduVideoIE(InfoExtractor): if category == 'tv': category = 'tvplay' - playlist_detail = self._call_api('xqinfo', category, playlist_id) + playlist_detail = self._call_api( + 'xqinfo', category, playlist_id, 'Download playlist JSON metadata') playlist_title = playlist_detail['title'] playlist_description = unescapeHTML(playlist_detail.get('intro')) - episodes_detail = self._call_api('xqsingle', category, playlist_id) + episodes_detail = self._call_api( + 'xqsingle', category, playlist_id, 'Download episodes JSON metadata') entries = [] for episode in episodes_detail['videos']: From b7546397f080e65fed42c69701f423e4f57f43f5 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:46:40 +0800 Subject: [PATCH 031/182] [baidu] Use list comprehension --- youtube_dl/extractor/baidu.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index d35116a7c..76b21e596 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -48,10 +48,9 @@ class BaiduVideoIE(InfoExtractor): episodes_detail = self._call_api( 'xqsingle', category, playlist_id, 'Download episodes JSON metadata') - entries = [] - for episode in episodes_detail['videos']: - entries.append(self.url_result( - episode['url'], video_title=episode['title'])) + entries = [self.url_result( + episode['url'], video_title=episode['title'] + ) for episode in episodes_detail['videos']] return self.playlist_result( entries, playlist_id, playlist_title, playlist_description) From 2fffb1dcd0790da264f86f4a8d9c54a127269aca Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 22:33:33 +0800 Subject: [PATCH 032/182] [qqmusic:playlist] Capture errors and update tests --- youtube_dl/extractor/qqmusic.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py index 1ba3bbddf..03f04b724 100644 --- a/youtube_dl/extractor/qqmusic.py +++ b/youtube_dl/extractor/qqmusic.py @@ -11,6 +11,7 @@ from ..utils import ( strip_jsonp, unescapeHTML, clean_html, + ExtractorError, ) @@ -315,7 +316,7 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE): IE_DESC = 'QQ音乐 - 歌单' _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)' - _TEST = { + _TESTS = [{ 'url': 'http://y.qq.com/#type=taoge&id=3462654915', 'info_dict': { 'id': '3462654915', @@ -323,7 +324,16 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE): 'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4', }, 'playlist_count': 40, - } + 'skip': 'playlist gone', + }, { + 'url': 'http://y.qq.com/#type=taoge&id=1374105607', + 'info_dict': { + 'id': '1374105607', + 'title': '易入人心的华语民谣', + 'description': '民谣的歌曲易于传唱、、歌词朗朗伤口、旋律简单温馨。属于那种才入耳孔。却上心头的感觉。没有太多的复杂情绪。简单而直接地表达乐者的情绪，就是这样的简单才易入人心。', + }, + 'playlist_count': 20, + }] def _real_extract(self, url): list_id = self._match_id(url) @@ -331,14 +341,21 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE): list_json = self._download_json( 'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s' % list_id, list_id, 'Download list page', - transform_source=strip_jsonp)['cdlist'][0] + transform_source=strip_jsonp) + if not len(list_json.get('cdlist', [])): + if list_json.get('code'): + raise ExtractorError( + 'QQ Music said: error %d in fetching playlist info' % list_json['code'], + expected=True) + raise ExtractorError('Unable to get playlist info') + cdlist = list_json['cdlist'][0] entries = [ self.url_result( 'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid'] - ) for song in list_json['songlist'] + ) for song in cdlist['songlist'] ] - list_name = list_json.get('dissname') - list_description = clean_html(unescapeHTML(list_json.get('desc'))) + list_name = cdlist.get('dissname') + list_description = clean_html(unescapeHTML(cdlist.get('desc'))) return self.playlist_result(entries, list_id, list_name, list_description) From 141a273a8b7a95fac5ccc0b5141c8d2eac48e3f9 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 22:39:09 +0800 Subject: [PATCH 033/182] [qqmusic] Update tests --- youtube_dl/extractor/qqmusic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py index 03f04b724..45a3c41c5 100644 --- a/youtube_dl/extractor/qqmusic.py +++ b/youtube_dl/extractor/qqmusic.py @@ -178,7 +178,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE): 'info_dict': { 'id': '001BLpXF2DyJe2', 'title': '林俊杰', - 'description': 'md5:2a222d89ba4455a3af19940c0481bb78', + 'description': 'md5:870ec08f7d8547c29c93010899103751', }, 'playlist_count': 12, } @@ -273,7 +273,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE): 'url': 'http://y.qq.com/#type=toplist&p=top_3', 'info_dict': { 'id': 'top_3', - 'title': 'QQ音乐巅峰榜·欧美', + 'title': '巅峰榜·欧美', 'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统' '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据' '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：' From 6b461026616b0e3ede9a302c74fc437541e19343 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Sat, 2 Jan 2016 21:24:57 +0100 Subject: [PATCH 034/182] [zdf] fix rtmpt format downloading handle errors --- youtube_dl/extractor/zdf.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index 2a1f2f6d1..c619a75e2 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -13,6 +13,7 @@ from ..utils import ( determine_ext, qualities, float_or_none, + ExtractorError, ) @@ -59,7 +60,6 @@ class ZDFIE(InfoExtractor): 'ext': 'flv', 'format_id': '%s-%d' % (proto, bitrate), 'tbr': bitrate, - 'protocol': proto, }) self._sort_formats(formats) return formats @@ -70,6 +70,15 @@ class ZDFIE(InfoExtractor): note='Downloading video info', errnote='Failed to download video info') + status_code = doc.find('./status/statuscode') + if status_code is not None and status_code.text != 'ok': + code = status_code.text + if code == 'notVisibleAnymore': + message = 'Video %s is not available' % video_id + else: + message = '%s returned error: %s' % (self.IE_NAME, code) + raise ExtractorError(message, expected=True) + title = doc.find('.//information/title').text description = xpath_text(doc, './/information/detail', 'description') duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration')) @@ -129,10 +138,10 @@ class ZDFIE(InfoExtractor): video_url, video_id, fatal=False)) elif ext == 'm3u8': formats.extend(self._extract_m3u8_formats( - video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)) + video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)) elif ext == 'f4m': formats.extend(self._extract_f4m_formats( - video_url, video_id, f4m_id='hds', fatal=False)) + video_url, video_id, f4m_id=format_id, fatal=False)) else: proto = format_m.group('proto').lower() From 4059eabd58dd5e68aa1229c0ba0ffce0262ef7d8 Mon Sep 17 00:00:00 2001 From: remitamine <remitamine@gmail.com> Date: Sat, 2 Jan 2016 21:29:10 +0100 Subject: [PATCH 035/182] [dreisat] use extract_from_xml_url from ZDFIE for info extraction(fixes #7680)(fixes #8104)(closes #8121) --- youtube_dl/extractor/dreisat.py | 59 ++------------------------------- 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py index 8ac8587be..028144f20 100644 --- a/youtube_dl/extractor/dreisat.py +++ b/youtube_dl/extractor/dreisat.py @@ -2,14 +2,10 @@ from __future__ import unicode_literals import re -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - unified_strdate, -) +from .zdf import ZDFIE -class DreiSatIE(InfoExtractor): +class DreiSatIE(ZDFIE): IE_NAME = '3sat' _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$' _TESTS = [ @@ -35,53 +31,4 @@ class DreiSatIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id - details_doc = self._download_xml(details_url, video_id, 'Downloading video details') - - status_code = details_doc.find('./status/statuscode') - if status_code is not None and status_code.text != 'ok': - code = status_code.text - if code == 'notVisibleAnymore': - message = 'Video %s is not available' % video_id - else: - message = '%s returned error: %s' % (self.IE_NAME, code) - raise ExtractorError(message, expected=True) - - thumbnail_els = details_doc.findall('.//teaserimage') - thumbnails = [{ - 'width': int(te.attrib['key'].partition('x')[0]), - 'height': int(te.attrib['key'].partition('x')[2]), - 'url': te.text, - } for te in thumbnail_els] - - information_el = details_doc.find('.//information') - video_title = information_el.find('./title').text - video_description = information_el.find('./detail').text - - details_el = details_doc.find('.//details') - video_uploader = details_el.find('./channel').text - upload_date = unified_strdate(details_el.find('./airtime').text) - - format_els = details_doc.findall('.//formitaet') - formats = [{ - 'format_id': fe.attrib['basetype'], - 'width': int(fe.find('./width').text), - 'height': int(fe.find('./height').text), - 'url': fe.find('./url').text, - 'filesize': int(fe.find('./filesize').text), - 'video_bitrate': int(fe.find('./videoBitrate').text), - } for fe in format_els - if not fe.find('./url').text.startswith('http://www.metafilegenerator.de/')] - - self._sort_formats(formats) - - return { - '_type': 'video', - 'id': video_id, - 'title': video_title, - 'formats': formats, - 'description': video_description, - 'thumbnails': thumbnails, - 'thumbnail': thumbnails[-1]['url'], - 'uploader': video_uploader, - 'upload_date': upload_date, - } + return self.extract_from_xml_url(video_id, details_url) From ea6abd740f8dd1c559e186bc1cbcfa73854a90d1 Mon Sep 17 00:00:00 2001 From: Sergey M <dstftw@gmail.com> Date: Sun, 3 Jan 2016 10:12:13 +0500 Subject: [PATCH 036/182] [nowtv] Mark broken --- youtube_dl/extractor/nowtv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py index fd107aca2..916a102bf 100644 --- a/youtube_dl/extractor/nowtv.py +++ b/youtube_dl/extractor/nowtv.py @@ -71,6 +71,7 @@ class NowTVBaseIE(InfoExtractor): class NowTVIE(NowTVBaseIE): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)' _TESTS = [{ From a7aaa39863a6b1ab89a56a55521fd4779c6ac900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 01:08:34 +0600 Subject: [PATCH 037/182] [utils] Extract known extensions for reuse --- youtube_dl/utils.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0ed6c45c8..da4ec7f20 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -70,6 +70,21 @@ ENGLISH_MONTH_NAMES = [ 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] +KNOWN_EXTENSIONS = ( + 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac', + 'flv', 'f4v', 'f4a', 'f4b', + 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus', + 'mkv', 'mka', 'mk3d', + 'avi', 'divx', + 'mov', + 'asf', 'wmv', 'wma', + '3gp', '3g2', + 'mp3', + 'flac', + 'ape', + 'wav', + 'f4f', 'f4m', 'm3u8', 'smil') + def preferredencoding(): """Get preferred encoding. @@ -942,20 +957,8 @@ def determine_ext(url, default_ext='unknown_video'): guess = url.partition('?')[0].rpartition('.')[2] if re.match(r'^[A-Za-z0-9]+$', guess): return guess - elif guess.rstrip('/') in ( - 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac', - 'flv', 'f4v', 'f4a', 'f4b', - 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus', - 'mkv', 'mka', 'mk3d', - 'avi', 'divx', - 'mov', - 'asf', 'wmv', 'wma', - '3gp', '3g2', - 'mp3', - 'flac', - 'ape', - 'wav', - 'f4f', 'f4m', 'm3u8', 'smil'): + # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download + elif guess.rstrip('/') in KNOWN_EXTENSIONS: return guess.rstrip('/') else: return default_ext From e54c44eeab9e088c344ec379cdc4b1fbc63ff324 Mon Sep 17 00:00:00 2001 From: pingtux <pingtux@users.noreply.github.com> Date: Sat, 2 Jan 2016 01:13:49 +0100 Subject: [PATCH 038/182] [20min.ch] Add new extractor (closes #5977) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/min20.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 youtube_dl/extractor/min20.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 4c7e5223d..64e1fd334 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -364,6 +364,7 @@ from .mdr import MDRIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE from .mgoon import MgoonIE +from .min20 import Min20IE from .minhateca import MinhatecaIE from .ministrygrid import MinistryGridIE from .miomio import MioMioIE diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py new file mode 100644 index 000000000..a3f91671e --- /dev/null +++ b/youtube_dl/extractor/min20.py @@ -0,0 +1,40 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class Min20IE(InfoExtractor): + _VALID_URL = r'http://www\.20min\.ch/.+?-(?P<id>[0-9]+)$' + _TEST = { + 'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469', + 'md5': 'cd4cbb99b94130cff423e967cd275e5e', + 'info_dict': { + 'id': '22050469', + 'ext': 'flv', + 'title': '«Wir müssen mutig nach vorne schauen»', + 'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.', + 'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg' + } + } + + # location of the flv videos, can't be extracted from the web page + _BASE_URL = "http://flv-rr.20min-tv.ch/videos/" + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'<h1><span>(.+?)</span></h1>', webpage, 'title') + flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id') + + description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description') + thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)" />', webpage, 'thumbnail') + url = self._BASE_URL + flash_id + "m.flv" + + return { + 'id': video_id, + 'url': url, + 'title': title, + 'description': description, + 'thumbnail': thumbnail + } From 66295fa4a6a712d4d41ff54ee9c361c7c83b44bc Mon Sep 17 00:00:00 2001 From: pingtux <pingtux@users.noreply.github.com> Date: Sat, 2 Jan 2016 16:19:05 +0100 Subject: [PATCH 039/182] [20min.ch] Added support for videoportal --- youtube_dl/extractor/min20.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py index a3f91671e..23aead19d 100644 --- a/youtube_dl/extractor/min20.py +++ b/youtube_dl/extractor/min20.py @@ -1,12 +1,14 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor class Min20IE(InfoExtractor): - _VALID_URL = r'http://www\.20min\.ch/.+?-(?P<id>[0-9]+)$' - _TEST = { + _VALID_URL = r'http://www\.20min\.ch/(videotv/\?vid=(?P<video_id>[0-9]+)|.+?-(?P<page_id>[0-9]+)$)' + _TESTS = [{ 'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469', 'md5': 'cd4cbb99b94130cff423e967cd275e5e', 'info_dict': { @@ -16,15 +18,29 @@ class Min20IE(InfoExtractor): 'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.', 'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg' } - } + }, { + 'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2', + 'md5': 'b52d6bc6ea6398e6a38f12cfd418149c', + 'info_dict': { + 'id': '469148', + 'ext': 'flv', + 'title': '85 000 Franken für 15 perfekte Minuten', + 'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)', + 'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg' + } + }] # location of the flv videos, can't be extracted from the web page _BASE_URL = "http://flv-rr.20min-tv.ch/videos/" def _real_extract(self, url): - video_id = self._match_id(url) + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('page_id') + if video_id is None: + # URL from the videoportal + video_id = mobj.group('video_id') webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'<h1><span>(.+?)</span></h1>', webpage, 'title') + title = self._html_search_regex(r'<h1>.*<span>(.+?)</span></h1>', webpage, 'title') flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id') description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description') From 133b1886fc6721090ea5d3be3d382626e2602b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 02:33:08 +0600 Subject: [PATCH 040/182] [20min] Improve (Closes #8110) --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/min20.py | 56 ------------------------ youtube_dl/extractor/twentymin.py | 73 +++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 57 deletions(-) delete mode 100644 youtube_dl/extractor/min20.py create mode 100644 youtube_dl/extractor/twentymin.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 64e1fd334..625b0bf16 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -364,7 +364,6 @@ from .mdr import MDRIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE from .mgoon import MgoonIE -from .min20 import Min20IE from .minhateca import MinhatecaIE from .ministrygrid import MinistryGridIE from .miomio import MioMioIE @@ -747,6 +746,7 @@ from .tvp import TvpIE, TvpSeriesIE from .tvplay import TVPlayIE from .tweakers import TweakersIE from .twentyfourvideo import TwentyFourVideoIE +from .twentymin import TwentyMinutenIE from .twentytwotracks import ( TwentyTwoTracksIE, TwentyTwoTracksGenreIE diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py deleted file mode 100644 index 23aead19d..000000000 --- a/youtube_dl/extractor/min20.py +++ /dev/null @@ -1,56 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor - - -class Min20IE(InfoExtractor): - _VALID_URL = r'http://www\.20min\.ch/(videotv/\?vid=(?P<video_id>[0-9]+)|.+?-(?P<page_id>[0-9]+)$)' - _TESTS = [{ - 'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469', - 'md5': 'cd4cbb99b94130cff423e967cd275e5e', - 'info_dict': { - 'id': '22050469', - 'ext': 'flv', - 'title': '«Wir müssen mutig nach vorne schauen»', - 'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.', - 'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg' - } - }, { - 'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2', - 'md5': 'b52d6bc6ea6398e6a38f12cfd418149c', - 'info_dict': { - 'id': '469148', - 'ext': 'flv', - 'title': '85 000 Franken für 15 perfekte Minuten', - 'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)', - 'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg' - } - }] - - # location of the flv videos, can't be extracted from the web page - _BASE_URL = "http://flv-rr.20min-tv.ch/videos/" - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('page_id') - if video_id is None: - # URL from the videoportal - video_id = mobj.group('video_id') - webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'<h1>.*<span>(.+?)</span></h1>', webpage, 'title') - flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id') - - description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description') - thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)" />', webpage, 'thumbnail') - url = self._BASE_URL + flash_id + "m.flv" - - return { - 'id': video_id, - 'url': url, - 'title': title, - 'description': description, - 'thumbnail': thumbnail - } diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py new file mode 100644 index 000000000..ca7d953b8 --- /dev/null +++ b/youtube_dl/extractor/twentymin.py @@ -0,0 +1,73 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import remove_end + + +class TwentyMinutenIE(InfoExtractor): + IE_NAME = '20min' + _VALID_URL = r'https?://(?:www\.)?20min\.ch/(?:videotv/*\?.*\bvid=(?P<id>\d+)|(?:[^/]+/)*(?P<display_id>[^/#?]+))' + _TESTS = [{ + # regular video + 'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2', + 'md5': 'b52d6bc6ea6398e6a38f12cfd418149c', + 'info_dict': { + 'id': '469148', + 'ext': 'flv', + 'title': '85 000 Franken für 15 perfekte Minuten', + 'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)', + 'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg' + } + }, { + # news article with video + 'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469', + 'md5': 'cd4cbb99b94130cff423e967cd275e5e', + 'info_dict': { + 'id': '469408', + 'display_id': '-Wir-muessen-mutig-nach-vorne-schauen--22050469', + 'ext': 'flv', + 'title': '«Wir müssen mutig nach vorne schauen»', + 'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.', + 'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg' + } + }, { + 'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738', + 'only_matching': True, + }, { + 'url': 'http://www.20min.ch/ro/sortir/cinema/story/Grandir-au-bahut--c-est-dur-18927411', + 'only_matching': True, + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + display_id = mobj.group('display_id') or video_id + + webpage = self._download_webpage(url, display_id) + + title = self._html_search_regex( + r'<h1>.*?<span>(.+?)</span></h1>', + webpage, 'title', default=None) + if not title: + title = remove_end(re.sub( + r'^20 [Mm]inuten.*? -', '', self._og_search_title(webpage)), ' - News') + + if not video_id: + video_id = self._search_regex( + r'"file\d?"\s*,\s*\"(\d+)', webpage, 'video id') + + description = self._html_search_meta( + 'description', webpage, 'description') + thumbnail = self._og_search_thumbnail(webpage) + + return { + 'id': video_id, + 'display_id': display_id, + 'url': 'http://speed.20min-tv.ch/%sm.flv' % video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + } From 5aa535c32982f1f9b2f689097a64be78d25e7d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 02:55:25 +0600 Subject: [PATCH 041/182] [bbccouk] Update tests (Closes #8090) --- youtube_dl/extractor/bbc.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py index 7b169881a..ce99a34ab 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dl/extractor/bbc.py @@ -124,14 +124,14 @@ class BBCCoUkIE(InfoExtractor): }, 'skip': 'Episode is no longer available on BBC iPlayer Radio', }, { - 'url': 'http://www.bbc.co.uk/music/clips/p02frcc3', + 'url': 'http://www.bbc.co.uk/music/clips/p022h44b', 'note': 'Audio', 'info_dict': { - 'id': 'p02frcch', + 'id': 'p022h44j', 'ext': 'flv', - 'title': 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix', - 'description': 'French house superstar Madeon takes us out of the club and onto the after party.', - 'duration': 3507, + 'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances', + 'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.", + 'duration': 227, }, 'params': { # rtmp download @@ -182,13 +182,12 @@ class BBCCoUkIE(InfoExtractor): }, { # iptv-all mediaset fails with geolocation however there is no geo restriction # for this programme at all - 'url': 'http://www.bbc.co.uk/programmes/b06bp7lf', + 'url': 'http://www.bbc.co.uk/programmes/b06rkn85', 'info_dict': { - 'id': 'b06bp7kf', + 'id': 'b06rkms3', 'ext': 'flv', - 'title': "Annie Mac's Friday Night, B.Traits sits in for Annie", - 'description': 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.', - 'duration': 10800, + 'title': "Best of the Mini-Mixes 2015: Part 3, Annie Mac's Friday Night - BBC Radio 1", + 'description': "Annie has part three in the Best of the Mini-Mixes 2015, plus the year's Most Played!", }, 'params': { # rtmp download From ab3176af3472917537635bd96b860a974094b0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 03:34:15 +0600 Subject: [PATCH 042/182] [ivi] Fix extraction and modernize --- youtube_dl/extractor/ivi.py | 69 ++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index 029878d24..216c534b5 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -7,6 +7,7 @@ import json from .common import InfoExtractor from ..utils import ( ExtractorError, + int_or_none, sanitized_Request, ) @@ -27,7 +28,7 @@ class IviIE(InfoExtractor): 'title': 'Иван Васильевич меняет профессию', 'description': 'md5:b924063ea1677c8fe343d8a72ac2195f', 'duration': 5498, - 'thumbnail': 'http://thumbs.ivi.ru/f20.vcp.digitalaccess.ru/contents/d/1/c3c885163a082c29bceeb7b5a267a6.jpg', + 'thumbnail': 're:^https?://.*\.jpg$', }, 'skip': 'Only works from Russia', }, @@ -38,33 +39,23 @@ class IviIE(InfoExtractor): 'info_dict': { 'id': '9549', 'ext': 'mp4', - 'title': 'Двое из ларца - Серия 1', + 'title': 'Двое из ларца - Дело Гольдберга (1 часть)', + 'series': 'Двое из ларца', + 'episode': 'Дело Гольдберга (1 часть)', + 'episode_number': 1, 'duration': 2655, - 'thumbnail': 'http://thumbs.ivi.ru/f15.vcp.digitalaccess.ru/contents/8/4/0068dc0677041f3336b7c2baad8fc0.jpg', + 'thumbnail': 're:^https?://.*\.jpg$', }, 'skip': 'Only works from Russia', } ] # Sorted by quality - _known_formats = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ'] - - # Sorted by size - _known_thumbnails = ['Thumb-120x90', 'Thumb-160', 'Thumb-640x480'] - - def _extract_description(self, html): - m = re.search(r'<meta name="description" content="(?P<description>[^"]+)"/>', html) - return m.group('description') if m is not None else None - - def _extract_comment_count(self, html): - m = re.search('(?s)<a href="#" id="view-comments" class="action-button dim gradient">\s*Комментарии:\s*(?P<commentcount>\d+)\s*</a>', html) - return int(m.group('commentcount')) if m is not None else 0 + _KNOWN_FORMATS = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ'] def _real_extract(self, url): video_id = self._match_id(url) - api_url = 'http://api.digitalaccess.ru/api/json/' - data = { 'method': 'da.content.get', 'params': [ @@ -76,11 +67,10 @@ class IviIE(InfoExtractor): ] } - request = sanitized_Request(api_url, json.dumps(data)) - - video_json_page = self._download_webpage( + request = sanitized_Request( + 'http://api.digitalaccess.ru/api/json/', json.dumps(data)) + video_json = self._download_json( request, video_id, 'Downloading video JSON') - video_json = json.loads(video_json_page) if 'error' in video_json: error = video_json['error'] @@ -95,35 +85,42 @@ class IviIE(InfoExtractor): formats = [{ 'url': x['url'], 'format_id': x['content_format'], - 'preference': self._known_formats.index(x['content_format']), - } for x in result['files'] if x['content_format'] in self._known_formats] + 'preference': self._KNOWN_FORMATS.index(x['content_format']), + } for x in result['files'] if x['content_format'] in self._KNOWN_FORMATS] self._sort_formats(formats) - if not formats: - raise ExtractorError('No media links available for %s' % video_id) - - duration = result['duration'] - compilation = result['compilation'] title = result['title'] + duration = int_or_none(result.get('duration')) + compilation = result.get('compilation') + episode = title if compilation else None + title = '%s - %s' % (compilation, title) if compilation is not None else title - previews = result['preview'] - previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format'])) - thumbnail = previews[-1]['url'] if len(previews) > 0 else None + thumbnails = [{ + 'url': preview['url'], + 'id': preview.get('content_format'), + } for preview in result.get('preview', []) if preview.get('url')] - video_page = self._download_webpage(url, video_id, 'Downloading video page') - description = self._extract_description(video_page) - comment_count = self._extract_comment_count(video_page) + webpage = self._download_webpage(url, video_id) + + episode_number = int_or_none(self._search_regex( + r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)', + webpage, 'episode number', default=None)) + + description = self._og_search_description(webpage, default=None) or self._html_search_meta( + 'description', webpage, 'description', default=None) return { 'id': video_id, 'title': title, - 'thumbnail': thumbnail, + 'series': compilation, + 'episode': episode, + 'episode_number': episode_number, + 'thumbnails': thumbnails, 'description': description, 'duration': duration, - 'comment_count': comment_count, 'formats': formats, } From c6270b2ed5be9e83bcb8114511641f9c5ad9008d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 03:49:18 +0600 Subject: [PATCH 043/182] [ivi:compilation] Fix extraction --- youtube_dl/extractor/ivi.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index 216c534b5..9ccfee173 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -146,8 +146,11 @@ class IviCompilationIE(InfoExtractor): }] def _extract_entries(self, html, compilation_id): - return [self.url_result('http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), 'Ivi') - for serie in re.findall(r'<strong><a href="/watch/%s/(\d+)">(?:[^<]+)</a></strong>' % compilation_id, html)] + return [ + self.url_result( + 'http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), IviIE.ie_key()) + for serie in re.findall( + r'<a href="/watch/%s/(\d+)"[^>]+data-id="\1"' % compilation_id, html)] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -155,7 +158,8 @@ class IviCompilationIE(InfoExtractor): season_id = mobj.group('seasonid') if season_id is not None: # Season link - season_page = self._download_webpage(url, compilation_id, 'Downloading season %s web page' % season_id) + season_page = self._download_webpage( + url, compilation_id, 'Downloading season %s web page' % season_id) playlist_id = '%s/season%s' % (compilation_id, season_id) playlist_title = self._html_search_meta('title', season_page, 'title') entries = self._extract_entries(season_page, compilation_id) @@ -163,8 +167,9 @@ class IviCompilationIE(InfoExtractor): compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page') playlist_id = compilation_id playlist_title = self._html_search_meta('title', compilation_page, 'title') - seasons = re.findall(r'<a href="/watch/%s/season(\d+)">[^<]+</a>' % compilation_id, compilation_page) - if len(seasons) == 0: # No seasons in this compilation + seasons = re.findall( + r'<a href="/watch/%s/season(\d+)' % compilation_id, compilation_page) + if not seasons: # No seasons in this compilation entries = self._extract_entries(compilation_page, compilation_id) else: entries = [] From 1463c5b9ac16bd50d5e30d2145ec584d1e6e74ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 03:54:52 +0600 Subject: [PATCH 044/182] [ivi] Extract season info --- youtube_dl/extractor/ivi.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index 9ccfee173..d0f00cdea 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -41,6 +41,8 @@ class IviIE(InfoExtractor): 'ext': 'mp4', 'title': 'Двое из ларца - Дело Гольдберга (1 часть)', 'series': 'Двое из ларца', + 'season': 'Сезон 1', + 'season_number': 1, 'episode': 'Дело Гольдберга (1 часть)', 'episode_number': 1, 'duration': 2655, @@ -105,6 +107,13 @@ class IviIE(InfoExtractor): webpage = self._download_webpage(url, video_id) + season = self._search_regex( + r'<li[^>]+class="season active"[^>]*><a[^>]+>([^<]+)', + webpage, 'season', default=None) + season_number = int_or_none(self._search_regex( + r'<li[^>]+class="season active"[^>]*><a[^>]+data-season(?:-index)?="(\d+)"', + webpage, 'season number', default=None)) + episode_number = int_or_none(self._search_regex( r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)', webpage, 'episode number', default=None)) @@ -116,6 +125,8 @@ class IviIE(InfoExtractor): 'id': video_id, 'title': title, 'series': compilation, + 'season': season, + 'season_number': season_number, 'episode': episode, 'episode_number': episode_number, 'thumbnails': thumbnails, From 896c7a23cd9fbfc589f26e9a7bbada38a325e2ad Mon Sep 17 00:00:00 2001 From: bpfoley <bpfoley> Date: Wed, 30 Dec 2015 21:50:26 +0000 Subject: [PATCH 045/182] [extractor/rte.py] Add support for RTE radio player While here, stop RteIE changing filename extensions to .mp4. The files saved are .flv containers with h264 video. --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/rte.py | 77 ++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index fb7151443..fe3c32874 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -556,7 +556,7 @@ from .ro220 import Ro220IE from .rottentomatoes import RottenTomatoesIE from .roxwel import RoxwelIE from .rtbf import RTBFIE -from .rte import RteIE +from .rte import RteIE, RteRadioIE from .rtlnl import RtlNlIE from .rtl2 import RTL2IE from .rtp import RTPIE diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py index d9cfbf180..25abcee92 100644 --- a/youtube_dl/extractor/rte.py +++ b/youtube_dl/extractor/rte.py @@ -5,16 +5,19 @@ from .common import InfoExtractor from ..utils import ( float_or_none, + unescapeHTML, ) class RteIE(InfoExtractor): + IE_NAME = 'rte' + IE_DESC = 'Raidió Teilifís Éireann TV' _VALID_URL = r'https?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)' _TEST = { 'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/', 'info_dict': { 'id': '10478715', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'Watch iWitness online', 'thumbnail': 're:^https?://.*\.jpg$', 'description': 'iWitness : The spirit of Ireland, one voice and one minute at a time.', @@ -44,13 +47,6 @@ class RteIE(InfoExtractor): # f4m_url = server + relative_url f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url'] f4m_formats = self._extract_f4m_formats(f4m_url, video_id) - f4m_formats = [{ - 'format_id': f['format_id'], - 'url': f['url'], - 'ext': 'mp4', - 'width': f['width'], - 'height': f['height'], - } for f in f4m_formats] return { 'id': video_id, @@ -60,3 +56,68 @@ class RteIE(InfoExtractor): 'thumbnail': thumbnail, 'duration': duration, } + + + +class RteRadioIE(InfoExtractor): + IE_NAME = 'rte:radio' + IE_DESC = 'Raidió Teilifís Éireann radio' + # Radioplayer URLs have the specifier #!rii=<channel_id>:<id>:<playable_item_id>:<date>: + # where the IDs are int/empty, the date is DD-MM-YYYY, and the specifier may be truncated. + # An <id> uniquely defines an individual recording, and is the only part we require. + _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:[0-9]*)(?:%3A|:)(?P<id>[0-9]+)' + + _TEST = { + 'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:', + 'info_dict': { + 'id': '10507902', + 'ext': 'flv', + 'title': 'Gloria', + 'thumbnail': 're:^https?://.*\.jpg$', + 'description': 'Tim Thurston guides you through a millennium of sacred music featuring Gregorian chant, pure solo voices and choral masterpieces, framed around the glorious music of J.S. Bach.', + 'duration': 7230.0, + }, + 'params': { + 'skip_download': 'f4m fails with --test atm' + } + } + + def _real_extract(self, url): + item_id = self._match_id(url) + feeds_url = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id + json_string = self._download_json(feeds_url, item_id) + + # NB the string values in the JSON are stored using XML escaping(!) + show = json_string['shows'][0] + title = unescapeHTML(show['title']) + description = unescapeHTML(show.get('description')) + thumbnail = show.get('thumbnail') + duration = float_or_none(show.get('duration'), 1000) + + mg = show['media:group'][0] + + formats = [] + + if mg.get('url') and not mg['url'].startswith('rtmpe:'): + formats.append({'url': mg.get('url')}) + + if mg.get('hls_server') and mg.get('hls_url'): + hls_url = mg['hls_server'] + mg['hls_url'] + hls_formats = self._extract_m3u8_formats( + hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False) + formats.extend(hls_formats) + + if mg.get('hds_server') and mg.get('hds_url'): + f4m_url = mg['hds_server'] + mg['hds_url'] + f4m_formats = self._extract_f4m_formats( + f4m_url, item_id, f4m_id='hds', fatal=False) + formats.extend(f4m_formats) + + return { + 'id': item_id, + 'title': title, + 'formats': formats, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + } From fb588f6a5663811da83cde68d17ff4eb041022ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 04:32:47 +0600 Subject: [PATCH 046/182] Credit @bpfoley for rte:radio (#8063) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 20f51009e..e39d07efe 100644 --- a/AUTHORS +++ b/AUTHORS @@ -150,3 +150,4 @@ reiv Muratcan Simsek Evan Lu flatgreen +Brian Foley From 0238451fc0f3c5f06827d5197a904981890a3d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 04:49:13 +0600 Subject: [PATCH 047/182] [rte] PEP 8 --- youtube_dl/extractor/rte.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py index 25abcee92..673399f04 100644 --- a/youtube_dl/extractor/rte.py +++ b/youtube_dl/extractor/rte.py @@ -58,7 +58,6 @@ class RteIE(InfoExtractor): } - class RteRadioIE(InfoExtractor): IE_NAME = 'rte:radio' IE_DESC = 'Raidió Teilifís Éireann radio' @@ -102,15 +101,15 @@ class RteRadioIE(InfoExtractor): formats.append({'url': mg.get('url')}) if mg.get('hls_server') and mg.get('hls_url'): - hls_url = mg['hls_server'] + mg['hls_url'] + hls_url = mg['hls_server'] + mg['hls_url'] hls_formats = self._extract_m3u8_formats( - hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False) + hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False) formats.extend(hls_formats) if mg.get('hds_server') and mg.get('hds_url'): f4m_url = mg['hds_server'] + mg['hds_url'] f4m_formats = self._extract_f4m_formats( - f4m_url, item_id, f4m_id='hds', fatal=False) + f4m_url, item_id, f4m_id='hds', fatal=False) formats.extend(f4m_formats) return { From 9746f4314ad0fe55076eb35fd70413623b0c8ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 05:01:32 +0600 Subject: [PATCH 048/182] [rte:radio] Simplify --- youtube_dl/extractor/rte.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py index 673399f04..c17ef2cc2 100644 --- a/youtube_dl/extractor/rte.py +++ b/youtube_dl/extractor/rte.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from .common import InfoExtractor - from ..utils import ( float_or_none, unescapeHTML, @@ -98,19 +97,19 @@ class RteRadioIE(InfoExtractor): formats = [] if mg.get('url') and not mg['url'].startswith('rtmpe:'): - formats.append({'url': mg.get('url')}) + formats.append({'url': mg['url']}) if mg.get('hls_server') and mg.get('hls_url'): - hls_url = mg['hls_server'] + mg['hls_url'] - hls_formats = self._extract_m3u8_formats( - hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False) - formats.extend(hls_formats) + formats.extend(self._extract_m3u8_formats( + mg['hls_server'] + mg['hls_url'], item_id, 'mp4', + entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)) if mg.get('hds_server') and mg.get('hds_url'): - f4m_url = mg['hds_server'] + mg['hds_url'] - f4m_formats = self._extract_f4m_formats( - f4m_url, item_id, f4m_id='hds', fatal=False) - formats.extend(f4m_formats) + formats.extend(self._extract_f4m_formats( + mg['hds_server'] + mg['hds_url'], item_id, + f4m_id='hds', fatal=False)) + + self._sort_formats(formats) return { 'id': item_id, From 9938a17f92ad8242d8841528a89df647ff759183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 4 Jan 2016 05:04:48 +0600 Subject: [PATCH 049/182] [rte:radio] Extract timestamp --- youtube_dl/extractor/rte.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py index c17ef2cc2..47c8331fe 100644 --- a/youtube_dl/extractor/rte.py +++ b/youtube_dl/extractor/rte.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( float_or_none, + parse_iso8601, unescapeHTML, ) @@ -69,10 +70,12 @@ class RteRadioIE(InfoExtractor): 'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:', 'info_dict': { 'id': '10507902', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Gloria', 'thumbnail': 're:^https?://.*\.jpg$', - 'description': 'Tim Thurston guides you through a millennium of sacred music featuring Gregorian chant, pure solo voices and choral masterpieces, framed around the glorious music of J.S. Bach.', + 'description': 'md5:9ce124a7fb41559ec68f06387cabddf0', + 'timestamp': 1451203200, + 'upload_date': '20151227', 'duration': 7230.0, }, 'params': { @@ -82,8 +85,10 @@ class RteRadioIE(InfoExtractor): def _real_extract(self, url): item_id = self._match_id(url) - feeds_url = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id - json_string = self._download_json(feeds_url, item_id) + + json_string = self._download_json( + 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id, + item_id) # NB the string values in the JSON are stored using XML escaping(!) show = json_string['shows'][0] @@ -91,6 +96,7 @@ class RteRadioIE(InfoExtractor): description = unescapeHTML(show.get('description')) thumbnail = show.get('thumbnail') duration = float_or_none(show.get('duration'), 1000) + timestamp = parse_iso8601(show.get('published')) mg = show['media:group'][0] @@ -114,8 +120,9 @@ class RteRadioIE(InfoExtractor): return { 'id': item_id, 'title': title, - 'formats': formats, 'description': description, 'thumbnail': thumbnail, + 'timestamp': timestamp, 'duration': duration, + 'formats': formats, } From 3f17c357d9958bfae18f06676c89ffdb63553509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 4 Jan 2016 18:35:31 +0100 Subject: [PATCH 050/182] [downloader/hls] Don't let ffmpeg read from stdin (#8139) If you run 'while read aurl ; do youtube-dl "${aurl}"; done < path_to_batch_file' (batch_file contains one url per line that uses the hls downloader) each call to youtube-dl consumed some characters and 'read' would assing to 'aurl' a non valid url (This is the same problem that was fixed for the ffmpeg postprocessors in cffcbc02de504d84e1c2677bb525c00b03e04f53) --- youtube_dl/downloader/hls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index b5a3e1167..d186d402d 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -46,7 +46,7 @@ class HlsFD(FileDownloader): self._debug_cmd(args) - retval = subprocess.call(args) + retval = subprocess.call(args, stdin=subprocess.PIPE) if retval == 0: fsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen('\r[%s] %s bytes' % (args[0], fsize)) From fff79f1867c5ac50eb7f88201dd3f728996d52a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:05:37 +0600 Subject: [PATCH 051/182] [amp] Add missing subtitles to info dict --- youtube_dl/extractor/amp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py index 1035d1c48..69e6baff7 100644 --- a/youtube_dl/extractor/amp.py +++ b/youtube_dl/extractor/amp.py @@ -76,5 +76,6 @@ class AMPIE(InfoExtractor): 'thumbnails': thumbnails, 'timestamp': parse_iso8601(item.get('pubDate'), ' '), 'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')), + 'subtitles': subtitles, 'formats': formats, } From 1dcc38b233f33112e00e1ddf0de6eb8ba1ef028e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:11:07 +0600 Subject: [PATCH 052/182] [dramafever] Improve subtitles extraction (Closes #8136) --- youtube_dl/extractor/dramafever.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py index b3b21d65f..0e9e6f7ba 100644 --- a/youtube_dl/extractor/dramafever.py +++ b/youtube_dl/extractor/dramafever.py @@ -105,13 +105,16 @@ class DramaFeverIE(DramaFeverBaseIE): video_id, 'Downloading episode info JSON', fatal=False) if episode_info: value = episode_info.get('value') - if value: - subfile = value[0].get('subfile') or value[0].get('new_subfile') - if subfile and subfile != 'http://www.dramafever.com/st/': - info.setdefault('subtitles', {}).setdefault('English', []).append({ - 'ext': 'srt', - 'url': subfile, - }) + if isinstance(value, list): + for v in value: + if v.get('type') == 'Episode': + subfile = v.get('subfile') or v.get('new_subfile') + if subfile and subfile != 'http://www.dramafever.com/st/': + info.setdefault('subtitles', {}).setdefault('English', []).append({ + 'ext': 'srt', + 'url': subfile, + }) + break return info From 8f4c56f33412db909e92218f4933d5b08beb9f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:17:33 +0600 Subject: [PATCH 053/182] [dramafever] Extract episode number --- youtube_dl/extractor/dramafever.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py index 0e9e6f7ba..8d71d6769 100644 --- a/youtube_dl/extractor/dramafever.py +++ b/youtube_dl/extractor/dramafever.py @@ -12,6 +12,7 @@ from ..compat import ( from ..utils import ( ExtractorError, clean_html, + int_or_none, sanitized_Request, ) @@ -114,6 +115,7 @@ class DramaFeverIE(DramaFeverBaseIE): 'ext': 'srt', 'url': subfile, }) + info['episode_number'] = int_or_none(v.get('number')) break return info From bd19aa0ed32ca4d041e691d47697ad7b82454b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:28:48 +0600 Subject: [PATCH 054/182] [dramafever] Extract episode --- youtube_dl/extractor/dramafever.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py index 8d71d6769..a14021a9e 100644 --- a/youtube_dl/extractor/dramafever.py +++ b/youtube_dl/extractor/dramafever.py @@ -115,7 +115,12 @@ class DramaFeverIE(DramaFeverBaseIE): 'ext': 'srt', 'url': subfile, }) - info['episode_number'] = int_or_none(v.get('number')) + episode_number = int_or_none(v.get('number')) + episode_fallback = 'Episode' + if episode_number: + episode_fallback += ' %d' % episode_number + info['episode'] = v.get('title', episode_fallback) + info['episode_number'] = episode_number break return info From a2e51e7b49a193b326291842d03aa49dd08326f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:36:38 +0600 Subject: [PATCH 055/182] [dramafever] Fix episode fallback --- youtube_dl/extractor/dramafever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py index a14021a9e..9f5e82f56 100644 --- a/youtube_dl/extractor/dramafever.py +++ b/youtube_dl/extractor/dramafever.py @@ -119,7 +119,7 @@ class DramaFeverIE(DramaFeverBaseIE): episode_fallback = 'Episode' if episode_number: episode_fallback += ' %d' % episode_number - info['episode'] = v.get('title', episode_fallback) + info['episode'] = v.get('title') or episode_fallback info['episode_number'] = episode_number break From 33cee6c7f6260dd33a9fbafd9f374c70454ff0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:41:18 +0600 Subject: [PATCH 056/182] [dramafever] Add test for custom episode title --- youtube_dl/extractor/dramafever.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py index 9f5e82f56..d35e88881 100644 --- a/youtube_dl/extractor/dramafever.py +++ b/youtube_dl/extractor/dramafever.py @@ -67,13 +67,15 @@ class DramaFeverBaseIE(AMPIE): class DramaFeverIE(DramaFeverBaseIE): IE_NAME = 'dramafever' _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)' - _TEST = { + _TESTS = [{ 'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/', 'info_dict': { 'id': '4512.1', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Cooking with Shin 4512.1', 'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0', + 'episode': 'Episode 1', + 'episode_number': 1, 'thumbnail': 're:^https?://.*\.jpg', 'timestamp': 1404336058, 'upload_date': '20140702', @@ -83,7 +85,25 @@ class DramaFeverIE(DramaFeverBaseIE): # m3u8 download 'skip_download': True, }, - } + }, { + 'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1', + 'info_dict': { + 'id': '4826.4', + 'ext': 'mp4', + 'title': 'Mnet Asian Music Awards 2015 4826.4', + 'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91', + 'episode': 'Mnet Asian Music Awards 2015 - Part 3', + 'episode_number': 4, + 'thumbnail': 're:^https?://.*\.jpg', + 'timestamp': 1450213200, + 'upload_date': '20151215', + 'duration': 5602, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }] def _real_extract(self, url): video_id = self._match_id(url).replace('/', '.') From 18c782ab26a53958a7d54d39e06eba0cd920178b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:58:25 +0600 Subject: [PATCH 057/182] [vrt] Extend _VALUD_URL --- youtube_dl/extractor/vrt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py index bbd3bbf7b..ee158b7b3 100644 --- a/youtube_dl/extractor/vrt.py +++ b/youtube_dl/extractor/vrt.py @@ -8,7 +8,7 @@ from ..utils import float_or_none class VRTIE(InfoExtractor): - _VALID_URL = r'https?://(?:deredactie|sporza|cobra)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*' + _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*' _TESTS = [ # deredactie.be { @@ -52,6 +52,10 @@ class VRTIE(InfoExtractor): 'duration': 661, } }, + { + 'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055', + 'only_matching': True, + } ] def _real_extract(self, url): From 2f546d0a3cc9b9fa7f022df68574f7009d7d1ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Tue, 5 Jan 2016 01:59:45 +0600 Subject: [PATCH 058/182] [vrt] Prefix format ids --- youtube_dl/extractor/vrt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py index ee158b7b3..01891ac4c 100644 --- a/youtube_dl/extractor/vrt.py +++ b/youtube_dl/extractor/vrt.py @@ -73,11 +73,11 @@ class VRTIE(InfoExtractor): if mobj: formats.extend(self._extract_m3u8_formats( '%s/%s' % (mobj.group('server'), mobj.group('path')), - video_id, 'mp4')) + video_id, 'mp4', m3u8_id='hls')) mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage) if mobj: formats.extend(self._extract_f4m_formats( - '%s/manifest.f4m' % mobj.group('src'), video_id)) + '%s/manifest.f4m' % mobj.group('src'), video_id, f4m_id='hds')) self._sort_formats(formats) title = self._og_search_title(webpage) From 40f796288afe634055c4600993b353da58e29a45 Mon Sep 17 00:00:00 2001 From: Sergey M <dstftw@gmail.com> Date: Tue, 5 Jan 2016 02:17:12 +0600 Subject: [PATCH 059/182] [README.md] Clarify cookies usage --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4fc83b8e3..75253199c 100644 --- a/README.md +++ b/README.md @@ -627,7 +627,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format. -Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. +Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare). ### Can you add support for this anime video site, or site which shows current movies for free? From 17b2d7ca772da5b709e00ca01e96e893807f7b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Wed, 6 Jan 2016 00:02:21 +0600 Subject: [PATCH 060/182] [udemy] Detect non free courses (Closes #8138) --- youtube_dl/extractor/udemy.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 1df636779..e19c1f762 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -11,6 +11,7 @@ from ..utils import ( float_or_none, int_or_none, sanitized_Request, + unescapeHTML, ) @@ -19,8 +20,6 @@ class UdemyIE(InfoExtractor): _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)' _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1' _ORIGIN_URL = 'https://www.udemy.com' - _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<' - _ALREADY_ENROLLED = '>You are already taking this course.<' _NETRC_MACHINE = 'udemy' _TESTS = [{ @@ -37,15 +36,21 @@ class UdemyIE(InfoExtractor): }] def _enroll_course(self, webpage, course_id): - enroll_url = self._search_regex( + checkout_url = unescapeHTML(self._search_regex( + r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/payment/checkout/.+?)\1', + webpage, 'checkout url', group='url', default=None)) + if checkout_url: + raise ExtractorError( + 'Course %s is not free. You have to pay for it before you can download.' + 'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True) + + enroll_url = unescapeHTML(self._search_regex( r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1', - webpage, 'enroll url', group='url', - default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id) - webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course') - if self._SUCCESSFULLY_ENROLLED in webpage: - self.to_screen('%s: Successfully enrolled in' % course_id) - elif self._ALREADY_ENROLLED in webpage: - self.to_screen('%s: Already enrolled in' % course_id) + webpage, 'enroll url', group='url', default=None)) + if enroll_url: + webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course') + if '>You have enrolled in' in webpage: + self.to_screen('%s: Successfully enrolled in the course' % course_id) def _download_lecture(self, course_id, lecture_id): return self._download_json( From f20756fb10ec560177282f032684327f600acc34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Wed, 6 Jan 2016 00:03:39 +0600 Subject: [PATCH 061/182] [udemy] Fix non free course message --- youtube_dl/extractor/udemy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index e19c1f762..f5b5e7fd6 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -41,7 +41,7 @@ class UdemyIE(InfoExtractor): webpage, 'checkout url', group='url', default=None)) if checkout_url: raise ExtractorError( - 'Course %s is not free. You have to pay for it before you can download.' + 'Course %s is not free. You have to pay for it before you can download. ' 'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True) enroll_url = unescapeHTML(self._search_regex( From 76048b23e8a4aac93a33a96356fe64a9bcf78421 Mon Sep 17 00:00:00 2001 From: kaspi <je326@hotmail.com> Date: Sat, 17 Oct 2015 23:27:03 -0400 Subject: [PATCH 062/182] [npr] Add extractor removed md5 from _TEST moved from xml data to json test changed _TEST url to one that will not expire, so tests would not be failing --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/npr.py | 71 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 youtube_dl/extractor/npr.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 19cea5e99..c46e19eae 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -473,6 +473,7 @@ from .npo import ( VPROIE, WNLIE ) +from .npr import NprIE from .nrk import ( NRKIE, NRKPlaylistIE, diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py new file mode 100644 index 000000000..a823bc096 --- /dev/null +++ b/youtube_dl/extractor/npr.py @@ -0,0 +1,71 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import os.path +import re + +from ..compat import compat_urllib_parse_unquote +from ..utils import url_basename +from .common import InfoExtractor + +class NprIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer.html?.*id=(?P<id>[0-9]+)' + _TEST = { + 'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205', + 'info_dict': { + 'id': '449974205', + 'ext': 'mp4', + 'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More' + } +} + + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage_url = 'http://www.npr.org/player/v2/mediaPlayer.html?id=' + video_id + webpage = self._download_webpage(webpage_url, video_id) + key = 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010' + xml_url = 'http://api.npr.org/query?id=%s&apiKey=%s' % (video_id, key) + json_url = 'http://api.npr.org/query?id=%s&apiKey=%s&format=json' % (video_id, key) + + formats = [] + entries = [] + + config = self._download_json(json_url, video_id) + + content = config["list"]["story"] + + album_title = config["list"]["story"][0]['song'][0]['album']['albumTitle'] + print album_title['$text'] + + for key in content: + if "audio" in key: + for x in key['audio']: + if x['type'] == 'standard': + playlist = True + song_duration = x["duration"]['$text'] + song_title = x["title"]["$text"] + song_id = x["id"] + + for k in x["format"]: + if type(x["format"][k]) is list: + for z in x["format"][k]: + formats.append({ 'format': z['type'], + 'url' : z['$text'] + }) + else: + formats.append({ 'format': k, + 'url' : x["format"][k]['$text'] + }) + + entries.append({ "title":song_title, + "id":song_id, + "duration": song_duration , + "formats":formats}) + formats = [] + + return { '_type': 'playlist', + 'id' : video_id, + 'title' : album_title, + 'entries': entries } From 51d3045de26fccbdcf0076c5cab8ab8152548bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 7 Jan 2016 01:57:36 +0600 Subject: [PATCH 063/182] [npr] Fix extractor (Closes #7218) --- youtube_dl/extractor/npr.py | 133 +++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py index a823bc096..125c7010b 100644 --- a/youtube_dl/extractor/npr.py +++ b/youtube_dl/extractor/npr.py @@ -1,71 +1,82 @@ -# coding: utf-8 from __future__ import unicode_literals -import os.path -import re - -from ..compat import compat_urllib_parse_unquote -from ..utils import url_basename from .common import InfoExtractor +from ..compat import compat_urllib_parse +from ..utils import ( + int_or_none, + qualities, +) + class NprIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer.html?.*id=(?P<id>[0-9]+)' - _TEST = { - 'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205', - 'info_dict': { - 'id': '449974205', - 'ext': 'mp4', - 'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More' - } -} - + _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)' + _TESTS = [{ + 'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205', + 'info_dict': { + 'id': '449974205', + 'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More' + }, + 'playlist_count': 7, + }, { + 'url': 'http://www.npr.org/player/v2/mediaPlayer.html?action=1&t=1&islist=false&id=446928052&m=446929930&live=1', + 'info_dict': { + 'id': '446928052', + 'title': "Songs We Love: Tigran Hamasyan, 'Your Mercy is Boundless'" + }, + 'playlist': [{ + 'md5': '12fa60cb2d3ed932f53609d4aeceabf1', + 'info_dict': { + 'id': '446929930', + 'ext': 'mp3', + 'title': 'Your Mercy is Boundless (Bazum en Qo gtutyunqd)', + 'duration': 402, + }, + }], + }] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - webpage_url = 'http://www.npr.org/player/v2/mediaPlayer.html?id=' + video_id - webpage = self._download_webpage(webpage_url, video_id) - key = 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010' - xml_url = 'http://api.npr.org/query?id=%s&apiKey=%s' % (video_id, key) - json_url = 'http://api.npr.org/query?id=%s&apiKey=%s&format=json' % (video_id, key) + playlist_id = self._match_id(url) + + config = self._download_json( + 'http://api.npr.org/query?%s' % compat_urllib_parse.urlencode({ + 'id': playlist_id, + 'fields': 'titles,audio,show', + 'format': 'json', + 'apiKey': 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010', + }), playlist_id) + + story = config['list']['story'][0] + + KNOWN_FORMATS = ('threegp', 'mp4', 'mp3') + quality = qualities(KNOWN_FORMATS) - formats = [] entries = [] + for audio in story.get('audio', []): + title = audio.get('title', {}).get('$text') + duration = int_or_none(audio.get('duration', {}).get('$text')) + formats = [] + for format_id, formats_entry in audio.get('format', {}).items(): + if not formats_entry: + continue + if isinstance(formats_entry, list): + formats_entry = formats_entry[0] + format_url = formats_entry.get('$text') + if not format_url: + continue + if format_id in KNOWN_FORMATS: + formats.append({ + 'url': format_url, + 'format_id': format_id, + 'ext': formats_entry.get('type'), + 'quality': quality(format_id), + }) + self._sort_formats(formats) + entries.append({ + 'id': audio['id'], + 'title': title, + 'duration': duration, + 'formats': formats, + }) - config = self._download_json(json_url, video_id) - - content = config["list"]["story"] - - album_title = config["list"]["story"][0]['song'][0]['album']['albumTitle'] - print album_title['$text'] - - for key in content: - if "audio" in key: - for x in key['audio']: - if x['type'] == 'standard': - playlist = True - song_duration = x["duration"]['$text'] - song_title = x["title"]["$text"] - song_id = x["id"] - - for k in x["format"]: - if type(x["format"][k]) is list: - for z in x["format"][k]: - formats.append({ 'format': z['type'], - 'url' : z['$text'] - }) - else: - formats.append({ 'format': k, - 'url' : x["format"][k]['$text'] - }) - - entries.append({ "title":song_title, - "id":song_id, - "duration": song_duration , - "formats":formats}) - formats = [] - - return { '_type': 'playlist', - 'id' : video_id, - 'title' : album_title, - 'entries': entries } + playlist_title = story.get('title', {}).get('$text') + return self.playlist_result(entries, playlist_id, playlist_title) From e4f49a87533bd6e3ebf11b43c3baf0e32db2f5ca Mon Sep 17 00:00:00 2001 From: oittaa <oittaa@users.noreply.github.com> Date: Thu, 7 Jan 2016 07:26:14 +0200 Subject: [PATCH 064/182] check video_play_path and use xpath_text "This check should take place earlier and should be more general if not video_url:. Same should be done for video_play_path. Also these fields better extracted with xpath_text." Suggestions by @dstftw --- youtube_dl/extractor/crunchyroll.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 00d943f77..785594df8 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -329,8 +329,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text streamdata_req, video_id, note='Downloading media info for %s' % video_format) stream_info = streamdata.find('./{default}preload/stream_info') - video_url = stream_info.find('./host').text - video_play_path = stream_info.find('./file').text + video_url = xpath_text(stream_info, './host') + video_play_path = xpath_text(stream_info, './file') + if not video_url or not video_play_path: + continue metadata = stream_info.find('./metadata') format_info = { 'format': video_format, From 2e02ecbccc9751af54ae2aab9ce973712c78af74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 7 Jan 2016 12:24:32 +0600 Subject: [PATCH 065/182] [ivideon] Add extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/ivideon.py | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 youtube_dl/extractor/ivideon.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index c46e19eae..a9d23b8f4 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -299,6 +299,7 @@ from .ivi import ( IviIE, IviCompilationIE ) +from .ivideon import IvideonIE from .izlesene import IzleseneIE from .jadorecettepub import JadoreCettePubIE from .jeuxvideo import JeuxVideoIE diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py new file mode 100644 index 000000000..aa05f5413 --- /dev/null +++ b/youtube_dl/extractor/ivideon.py @@ -0,0 +1,75 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_urllib_parse +from ..utils import qualities + + +class IvideonIE(InfoExtractor): + IE_NAME = 'ivideon' + IE_DESC = 'Ivideon TV' + _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)' + _TESTS = [{ + 'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/', + 'info_dict': { + 'id': '100-916ca13b5c4ad9f564266424a026386d', + 'ext': 'flv', + 'title': 're:^Касса [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', + 'description': 'Основное предназначение - запись действий кассиров. Плюс общий вид.', + 'is_live': True, + }, + 'params': { + 'skip_download': True, + } + }, { + 'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru', + 'only_matching': True, + }] + + _QUALITIES = ('low', 'mid', 'hi') + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + server_id, camera_id = mobj.group('id'), mobj.group('camera_id') + camera_name, description = None, None + + webpage = self._download_webpage(url, server_id, fatal=False) + if webpage: + config_string = self._search_regex( + r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None) + if config_string: + config = self._parse_json(config_string, server_id, fatal=False) + camera_info = config.get('ivTvAppOptions', {}).get('currentCameraInfo') + if camera_info: + camera_name = camera_info.get('camera_name') + description = camera_info.get('misc', {}).get('description') + if not camera_name: + camera_name = self._html_search_meta( + 'name', webpage, 'camera name', default=None) or self._search_regex( + r'<h1[^>]+class="b-video-title"[^>]*>([^<]+)', webpage, 'camera name', default=None) + + quality = qualities(self._QUALITIES) + + formats = [{ + 'url': 'https://streaming.ivideon.com/flv/live?%s' % compat_urllib_parse.urlencode({ + 'server': server_id, + 'camera': camera_id, + 'sessionId': 'demo', + 'q': quality(format_id), + }), + 'format_id': format_id, + 'ext': 'flv', + 'quality': quality(format_id), + } for format_id in self._QUALITIES] + self._sort_formats(formats) + + return { + 'id': server_id, + 'title': self._live_title(camera_name or server_id), + 'description': description, + 'is_live': True, + 'formats': formats, + } From 23f13e97548cc4ff3d11408ee5bc77f682e642dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 8 Jan 2016 00:52:55 +0600 Subject: [PATCH 066/182] [youtube] Support expanding alternative format of links in description (Closes #8164) --- youtube_dl/extractor/youtube.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 4aac2cc03..897641e42 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1235,10 +1235,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_description = re.sub(r'''(?x) <a\s+ (?:[a-zA-Z-]+="[^"]+"\s+)*? - title="([^"]+)"\s+ + (?:title|href)="([^"]+)"\s+ (?:[a-zA-Z-]+="[^"]+"\s+)*? - class="yt-uix-redirect-link"\s*> - [^<]+ + class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)".*?> + [^<]+\.{3}\s* </a> ''', r'\1', video_description) video_description = clean_html(video_description) From 97afd99a18e4723e4ff588df456c7aec62967b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 8 Jan 2016 01:54:31 +0600 Subject: [PATCH 067/182] [soundcloud:likes] Adapt to API changes (Closes #8166) --- youtube_dl/extractor/soundcloud.py | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 02e64e094..b2d5487ca 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -384,27 +384,24 @@ class SoundcloudUserIE(SoundcloudIE): resource = mobj.group('rsrc') or 'all' base_url = self._BASE_URL_MAP[resource] % user['id'] - next_href = None + COMMON_QUERY = { + 'limit': 50, + 'client_id': self._CLIENT_ID, + 'linked_partitioning': '1', + } + + query = COMMON_QUERY.copy() + query['offset'] = 0 + + next_href = base_url + '?' + compat_urllib_parse.urlencode(query) entries = [] for i in itertools.count(): - if not next_href: - data = compat_urllib_parse.urlencode({ - 'offset': i * 50, - 'limit': 50, - 'client_id': self._CLIENT_ID, - 'linked_partitioning': '1', - 'representation': 'speedy', - }) - next_href = base_url + '?' + data - response = self._download_json( next_href, uploader, 'Downloading track page %s' % (i + 1)) collection = response['collection'] - if not collection: - self.to_screen('%s: End page received' % uploader) break def resolve_permalink_url(candidates): @@ -419,12 +416,15 @@ class SoundcloudUserIE(SoundcloudIE): if permalink_url: entries.append(self.url_result(permalink_url)) - if 'next_href' in response: - next_href = response['next_href'] - if not next_href: - break - else: - next_href = None + next_href = response.get('next_href') + if not next_href: + break + + parsed_next_href = compat_urlparse.urlparse(response['next_href']) + qs = compat_urlparse.parse_qs(parsed_next_href.query) + qs.update(COMMON_QUERY) + next_href = compat_urlparse.urlunparse( + parsed_next_href._replace(query=compat_urllib_parse.urlencode(qs, True))) return { '_type': 'playlist', From 18e6c97c48f883911649d9b3d64127379a8b1df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 8 Jan 2016 03:19:47 +0600 Subject: [PATCH 068/182] [adultswim] Skip georestricted hls (Closes #8168) --- youtube_dl/extractor/adultswim.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py index bf21a6887..8157da2cb 100644 --- a/youtube_dl/extractor/adultswim.py +++ b/youtube_dl/extractor/adultswim.py @@ -187,7 +187,8 @@ class AdultSwimIE(InfoExtractor): media_url = file_el.text if determine_ext(media_url) == 'm3u8': formats.extend(self._extract_m3u8_formats( - media_url, segment_title, 'mp4', preference=0, m3u8_id='hls')) + media_url, segment_title, 'mp4', preference=0, + m3u8_id='hls', fatal=False)) else: formats.append({ 'format_id': '%s_%s' % (bitrate, ftype), From 4cf096a4a99c2cc29708f23244e4433d91c83b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 8 Jan 2016 05:11:23 +0600 Subject: [PATCH 069/182] [ivideon] Add support for map bound URLs --- youtube_dl/extractor/ivideon.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py index aa05f5413..617dc8c07 100644 --- a/youtube_dl/extractor/ivideon.py +++ b/youtube_dl/extractor/ivideon.py @@ -4,14 +4,17 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_urllib_parse +from ..compat import ( + compat_urllib_parse, + compat_urlparse, +) from ..utils import qualities class IvideonIE(InfoExtractor): IE_NAME = 'ivideon' IE_DESC = 'Ivideon TV' - _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)' + _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/(?:[^/]+/)*camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)' _TESTS = [{ 'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/', 'info_dict': { @@ -27,6 +30,9 @@ class IvideonIE(InfoExtractor): }, { 'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru', 'only_matching': True, + }, { + 'url': 'https://www.ivideon.com/tv/map/22.917923/-31.816406/16/camera/100-e7bc16c7d4b5bbd633fd5350b66dfa9a/0', + 'only_matching': True, }] _QUALITIES = ('low', 'mid', 'hi') @@ -35,8 +41,10 @@ class IvideonIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) server_id, camera_id = mobj.group('id'), mobj.group('camera_id') camera_name, description = None, None + camera_url = compat_urlparse.urljoin( + url, '/tv/camera/%s/%s/' % (server_id, camera_id)) - webpage = self._download_webpage(url, server_id, fatal=False) + webpage = self._download_webpage(camera_url, server_id, fatal=False) if webpage: config_string = self._search_regex( r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None) From 5dbe81a1d35ae704b5ea208698a6bb785923d71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 8 Jan 2016 10:41:24 +0600 Subject: [PATCH 070/182] [vimeo] Automatically pickup full movie when rented (Closes #8171) --- youtube_dl/extractor/vimeo.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 7af699982..776e8cce4 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -217,7 +217,7 @@ class VimeoIE(VimeoBaseInfoExtractor): r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage) if mobj: player_url = unescapeHTML(mobj.group('url')) - surl = smuggle_url(player_url, {'Referer': url}) + surl = smuggle_url(player_url, {'http_headers': {'Referer': url}}) return surl # Look for embedded (swf embed) Vimeo player mobj = re.search( @@ -262,11 +262,11 @@ class VimeoIE(VimeoBaseInfoExtractor): self._login() def _real_extract(self, url): - url, data = unsmuggle_url(url) + url, data = unsmuggle_url(url, {}) headers = std_headers - if data is not None: + if 'http_headers' in data: headers = headers.copy() - headers.update(data) + headers.update(data['http_headers']) if 'Referer' not in headers: headers['Referer'] = url @@ -342,7 +342,7 @@ class VimeoIE(VimeoBaseInfoExtractor): raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option') if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None: - if data and '_video_password_verified' in data: + if '_video_password_verified' in data: raise ExtractorError('video password verification failed!') self._verify_video_password(url, video_id, webpage) return self._real_extract( @@ -354,6 +354,13 @@ class VimeoIE(VimeoBaseInfoExtractor): if config.get('view') == 4: config = self._verify_player_video_password(url, video_id) + if '>You rented this title.<' in webpage: + feature_id = config.get('video', {}).get('vod', {}).get('feature_id') + if feature_id and not data.get('force_feature_id', False): + return self.url_result(smuggle_url( + 'https://player.vimeo.com/player/%s' % feature_id, + {'force_feature_id': True}), 'Vimeo') + # Extract title video_title = config["video"]["title"] From 5777f5d386d14407a19bc86c31e0bf2b5ae4a87f Mon Sep 17 00:00:00 2001 From: atomic83 <atomic83@riseup.net> Date: Fri, 8 Jan 2016 12:58:05 +0100 Subject: [PATCH 071/182] Extract xHamster title fix --- youtube_dl/extractor/xhamster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 8938c0e45..261d323a7 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -65,7 +65,7 @@ class XHamsterIE(InfoExtractor): title = self._html_search_regex( [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)', - r'

([^<]+)

'], webpage, 'title') + r'([^<]+)'], webpage, 'title') # Only a few videos have an description mobj = re.search(r'Description: ([^<]+)', webpage) From 6609b3ce371d08c0464a3caa30c21b61de221508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:19:36 +0600 Subject: [PATCH 072/182] [xhamster] Improve title extraction --- youtube_dl/extractor/xhamster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 261d323a7..97355d17f 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -64,8 +64,9 @@ class XHamsterIE(InfoExtractor): webpage = self._download_webpage(mrss_url, video_id) title = self._html_search_regex( - [r'(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)', - r'([^<]+)'], webpage, 'title') + [r']*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)', + r']*>([^<]+)', + r']+itemprop=".*?caption.*?"[^>]+content="(.+?)"'], webpage, 'title') # Only a few videos have an description mobj = re.search(r'Description: ([^<]+)', webpage) From 4763b624a6655bc2333157031c73858f6b918f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:21:57 +0600 Subject: [PATCH 073/182] [xhamster] Fix upload date extraction --- youtube_dl/extractor/xhamster.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 97355d17f..560c38e26 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -72,10 +72,9 @@ class XHamsterIE(InfoExtractor): mobj = re.search(r'Description: ([^<]+)', webpage) description = mobj.group(1) if mobj else None - upload_date = self._html_search_regex(r'hint=\'(\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}\'', - webpage, 'upload date', fatal=False) - if upload_date: - upload_date = unified_strdate(upload_date) + upload_date = unified_strdate(self._search_regex( + r'hint=["\'](\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}', + webpage, 'upload date', fatal=False)) uploader = self._html_search_regex( r"]+>(?P[^<]+)", From 44731e308cdcc89fda1e613e094aa23de33f5c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:26:37 +0600 Subject: [PATCH 074/182] [xhamster] Fix duration extraction --- youtube_dl/extractor/xhamster.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 560c38e26..2b9ac2419 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -4,10 +4,10 @@ import re from .common import InfoExtractor from ..utils import ( - unified_strdate, - str_to_int, + float_or_none, int_or_none, - parse_duration, + str_to_int, + unified_strdate, ) @@ -85,8 +85,9 @@ class XHamsterIE(InfoExtractor): r''']+poster=(?P["'])(?P.+?)(?P=q)[^>]*>'''], webpage, 'thumbnail', fatal=False, group='thumbnail') - duration = parse_duration(self._html_search_regex(r'Runtime: (\d+:\d+)', - webpage, 'duration', fatal=False)) + duration = float_or_none(self._search_regex( + r'(["\'])duration\1\s*:\s*(["\'])(?P.+?)\2', + webpage, 'duration', fatal=False, group='duration')) view_count = self._html_search_regex(r'Views: ([^<]+)', webpage, 'view count', fatal=False) if view_count: From 6a16fd4a1ad1dbd9372f75ddffce1e9fe95b002c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:29:10 +0600 Subject: [PATCH 075/182] [xhamster] Fix view count extraction --- youtube_dl/extractor/xhamster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 2b9ac2419..ccee77359 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -89,9 +89,9 @@ class XHamsterIE(InfoExtractor): r'(["\'])duration\1\s*:\s*(["\'])(?P.+?)\2', webpage, 'duration', fatal=False, group='duration')) - view_count = self._html_search_regex(r'Views: ([^<]+)', webpage, 'view count', fatal=False) - if view_count: - view_count = str_to_int(view_count) + view_count = int_or_none(self._search_regex( + r'content=["\']User(?:View|Play)s:(\d+)', + webpage, 'view count', fatal=False)) mobj = re.search(r"hint='(?P\d+) Likes / (?P\d+) Dislikes'", webpage) (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None) From 1a6d92847f4e5fe1d94f2f159f903a414d5ff62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:31:24 +0600 Subject: [PATCH 076/182] [xhamster] Change title regex precedence --- youtube_dl/extractor/xhamster.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index ccee77359..c9108c345 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -64,9 +64,10 @@ class XHamsterIE(InfoExtractor): webpage = self._download_webpage(mrss_url, video_id) title = self._html_search_regex( - [r']*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)', - r']*>([^<]+)', - r']+itemprop=".*?caption.*?"[^>]+content="(.+?)"'], webpage, 'title') + [r']*>([^<]+)', + r']+itemprop=".*?caption.*?"[^>]+content="(.+?)"', + r']*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)'], + webpage, 'title') # Only a few videos have an description mobj = re.search(r'Description: ([^<]+)', webpage) From bcac2a071040fd89585a806ca66e086c109406c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:36:19 +0600 Subject: [PATCH 077/182] [xhamster] Fix uploader extraction --- youtube_dl/extractor/xhamster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index c9108c345..981ee284f 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -78,7 +78,7 @@ class XHamsterIE(InfoExtractor): webpage, 'upload date', fatal=False)) uploader = self._html_search_regex( - r"]+>(?P[^<]+)", + r']+itemprop=["\']author[^>]+>]+href=["\'].+?xhamster\.com/user/[^>]+>(?P.+?)', webpage, 'uploader', default='anonymous') thumbnail = self._search_regex( From 14b4f038c0123138a642a25bf6a3851dd610d1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 00:36:43 +0600 Subject: [PATCH 078/182] [xhamster] Update tests --- youtube_dl/extractor/xhamster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 981ee284f..066af184f 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -22,7 +22,7 @@ class XHamsterIE(InfoExtractor): 'title': 'FemaleAgent Shy beauty takes the bait', 'upload_date': '20121014', 'uploader': 'Ruseful2011', - 'duration': 893, + 'duration': 893.52, 'age_limit': 18, } }, @@ -34,7 +34,7 @@ class XHamsterIE(InfoExtractor): 'title': 'Britney Spears Sexy Booty', 'upload_date': '20130914', 'uploader': 'jojo747400', - 'duration': 200, + 'duration': 200.48, 'age_limit': 18, } }, From d5f071afb5ce302058d99a14989f3f6afb88b1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 03:06:09 +0600 Subject: [PATCH 079/182] [vimeo] Check source file URL (Closes #8187) --- youtube_dl/extractor/vimeo.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 776e8cce4..5cb01907b 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -11,6 +11,7 @@ from ..compat import ( compat_urlparse, ) from ..utils import ( + determine_ext, encode_dict, ExtractorError, InAdvancePagedList, @@ -419,16 +420,21 @@ class VimeoIE(VimeoBaseInfoExtractor): download_data = self._download_json(download_request, video_id, fatal=False) if download_data: source_file = download_data.get('source_file') - if source_file and not source_file.get('is_cold') and not source_file.get('is_defrosting'): - formats.append({ - 'url': source_file['download_url'], - 'ext': source_file['extension'].lower(), - 'width': int_or_none(source_file.get('width')), - 'height': int_or_none(source_file.get('height')), - 'filesize': parse_filesize(source_file.get('size')), - 'format_id': source_file.get('public_name', 'Original'), - 'preference': 1, - }) + if isinstance(source_file, dict): + download_url = source_file.get('download_url') + if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'): + source_name = source_file.get('public_name', 'Original') + if self._is_valid_url(download_url, video_id, '%s video' % source_name): + ext = source_file.get('extension', determine_ext(download_url)).lower(), + formats.append({ + 'url': download_url, + 'ext': ext, + 'width': int_or_none(source_file.get('width')), + 'height': int_or_none(source_file.get('height')), + 'filesize': parse_filesize(source_file.get('size')), + 'format_id': source_name, + 'preference': 1, + }) config_files = config['video'].get('files') or config['request'].get('files', {}) for f in config_files.get('progressive', []): video_url = f.get('url') From 16f1131a4d46c4437485d4b075e9d1243ca8d60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 9 Jan 2016 03:07:29 +0600 Subject: [PATCH 080/182] [vimeo] Add test for #8187 --- youtube_dl/extractor/vimeo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 5cb01907b..76e681bc3 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -209,6 +209,11 @@ class VimeoIE(VimeoBaseInfoExtractor): 'url': 'https://vimeo.com/groups/travelhd/videos/22439234', 'only_matching': True, }, + { + # source file returns 403: Forbidden + 'url': 'https://vimeo.com/7809605', + 'only_matching': True, + }, ] @staticmethod From b374af6ebdf6298ab1593bee56dba5b0a1daf9b7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 9 Jan 2016 01:16:08 +0100 Subject: [PATCH 081/182] release 2016.01.09 --- docs/supportedsites.md | 10 +++++++--- youtube_dl/version.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 84c166805..8d0c7b97a 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1,6 +1,7 @@ # Supported sites - **1tv**: Первый канал - **1up.com** + - **20min** - **220.ro** - **22tracks:genre** - **22tracks:track** @@ -255,6 +256,7 @@ - **Ir90Tv** - **ivi**: ivi.ru - **ivi:compilation**: ivi.ru compilations + - **ivideon**: Ivideon TV - **Izlesene** - **JadoreCettePub** - **JeuxVideo** @@ -386,13 +388,14 @@ - **nowness** - **nowness:playlist** - **nowness:series** - - **NowTV** + - **NowTV** (Currently broken) - **NowTVList** - **nowvideo**: NowVideo - **npo**: npo.nl and ntr.nl - **npo.nl:live** - **npo.nl:radio** - **npo.nl:radio:fragment** + - **Npr** - **NRK** - **NRKPlaylist** - **NRKTV**: NRK TV and NRK Radio @@ -464,11 +467,13 @@ - **RegioTV** - **Restudy** - **ReverbNation** + - **Revision3** - **RingTV** - **RottenTomatoes** - **Roxwel** - **RTBF** - - **Rte** + - **rte**: Raidió Teilifís Éireann TV + - **rte:radio**: Raidió Teilifís Éireann radio - **rtl.nl**: rtl.nl and rtlxl.nl - **RTL2** - **RTP** @@ -573,7 +578,6 @@ - **TeleMB** - **TeleTask** - **TenPlay** - - **TestTube** - **TF1** - **TheIntercept** - **TheOnion** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 790bd5b3b..7030903c0 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2016.01.01' +__version__ = '2016.01.09' From 27783821af5a85fb20266834cff4f07eb0695cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 9 Jan 2016 11:16:23 +0100 Subject: [PATCH 082/182] [xhamster] Remove unused import --- youtube_dl/extractor/xhamster.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 066af184f..fd43e8854 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -6,7 +6,6 @@ from .common import InfoExtractor from ..utils import ( float_or_none, int_or_none, - str_to_int, unified_strdate, ) From 7a34302e95ee66f770b1eba720847831c40121a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 10 Jan 2016 01:37:10 +0600 Subject: [PATCH 083/182] [canalc2] Fix extraction (Closes #8191) --- youtube_dl/extractor/canalc2.py | 46 +++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py index f6a1ff381..f1f128c45 100644 --- a/youtube_dl/extractor/canalc2.py +++ b/youtube_dl/extractor/canalc2.py @@ -9,9 +9,9 @@ from ..utils import parse_duration class Canalc2IE(InfoExtractor): IE_NAME = 'canalc2.tv' - _VALID_URL = r'https?://(?:www\.)?canalc2\.tv/video/(?P\d+)' + _VALID_URL = r'https?://(?:(?:www\.)?canalc2\.tv/video/|archives-canalc2\.u-strasbg\.fr/video\.asp\?.*\bidVideo=)(?P\d+)' - _TEST = { + _TESTS = [{ 'url': 'http://www.canalc2.tv/video/12163', 'md5': '060158428b650f896c542dfbb3d6487f', 'info_dict': { @@ -23,24 +23,36 @@ class Canalc2IE(InfoExtractor): 'params': { 'skip_download': True, # Requires rtmpdump } - } + }, { + 'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui', + 'only_matching': True, + }] def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - video_url = self._search_regex( - r'jwplayer\((["\'])Player\1\)\.setup\({[^}]*file\s*:\s*(["\'])(?P.+?)\2', - webpage, 'video_url', group='file') - formats = [{'url': video_url}] - if video_url.startswith('rtmp://'): - rtmp = re.search(r'^(?Prtmp://[^/]+/(?P.+/))(?Pmp4:.+)$', video_url) - formats[0].update({ - 'url': rtmp.group('url'), - 'ext': 'flv', - 'app': rtmp.group('app'), - 'play_path': rtmp.group('play_path'), - 'page_url': url, - }) + + webpage = self._download_webpage( + 'http://www.canalc2.tv/video/%s' % video_id, video_id) + + formats = [] + for _, video_url in re.findall(r'file\s*=\s*(["\'])(.+?)\1', webpage): + if video_url.startswith('rtmp://'): + rtmp = re.search( + r'^(?Prtmp://[^/]+/(?P.+/))(?Pmp4:.+)$', video_url) + formats.append({ + 'url': rtmp.group('url'), + 'format_id': 'rtmp', + 'ext': 'flv', + 'app': rtmp.group('app'), + 'play_path': rtmp.group('play_path'), + 'page_url': url, + }) + else: + formats.append({ + 'url': video_url, + 'format_id': 'http', + }) + self._sort_formats(formats) title = self._html_search_regex( r'(?s)class="[^"]*col_description[^"]*">.*?

(.*?)

', webpage, 'title') From 0a899a1448328648927e18e43c6e2274d2706396 Mon Sep 17 00:00:00 2001 From: Vignesh Venkat Date: Sat, 9 Jan 2016 15:31:50 -0800 Subject: [PATCH 084/182] [Bigflix] Add new extractor for bigflix.com Add an IE to support bigflix.com. It uses some sort of silverlight plugin whose video url is being populated using base64 encoded flashvars. So it is quite straightforward to extract. --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/bigflix.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 youtube_dl/extractor/bigflix.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a9d23b8f4..40c42d4d2 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -61,6 +61,7 @@ from .beeg import BeegIE from .behindkink import BehindKinkIE from .beatportpro import BeatportProIE from .bet import BetIE +from .bigflix import BigflixIE from .bild import BildIE from .bilibili import BiliBiliIE from .bleacherreport import ( diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py new file mode 100644 index 000000000..aeea1a002 --- /dev/null +++ b/youtube_dl/extractor/bigflix.py @@ -0,0 +1,42 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from base64 import b64decode + +from .common import InfoExtractor +from ..compat import compat_urllib_parse_unquote + + +class BigflixIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.*/(?P[0-9]+)' + _TEST = { + 'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537', + 'md5': 'ec76aa9b1129e2e5b301a474e54fab74', + 'info_dict': { + 'id': '16537', + 'ext': 'mp4', + 'title': 'Singham Returns', + 'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d', + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex( + r']+class=["\']pagetitle["\'][^>]*>(.+?)', + webpage, 'title') + + video_url = b64decode(compat_urllib_parse_unquote(self._search_regex( + r'file=([^&]+)', webpage, 'video url')).encode('ascii')).decode('utf-8') + + description = self._html_search_meta('description', webpage) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'description': description, + } From 15b1c6656fd8b384bd974f4d943a04b8bf8ca915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 10 Jan 2016 10:03:56 +0600 Subject: [PATCH 085/182] Credit @vickyg3 for bigflix (#8194) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index e39d07efe..3d8bebbb0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -151,3 +151,4 @@ Muratcan Simsek Evan Lu flatgreen Brian Foley +Vignesh Venkat From 6e99d5762a844b44cad4cae144045ed0537c084b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 10 Jan 2016 10:31:36 +0600 Subject: [PATCH 086/182] [bigflix] Extract all formats --- youtube_dl/extractor/bigflix.py | 42 +++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py index aeea1a002..b7e498436 100644 --- a/youtube_dl/extractor/bigflix.py +++ b/youtube_dl/extractor/bigflix.py @@ -1,15 +1,16 @@ # coding: utf-8 from __future__ import unicode_literals -from base64 import b64decode +import base64 +import re from .common import InfoExtractor from ..compat import compat_urllib_parse_unquote class BigflixIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.*/(?P[0-9]+)' - _TEST = { + _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.+/(?P[0-9]+)' + _TESTS = [{ 'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537', 'md5': 'ec76aa9b1129e2e5b301a474e54fab74', 'info_dict': { @@ -18,7 +19,20 @@ class BigflixIE(InfoExtractor): 'title': 'Singham Returns', 'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d', } - } + }, { + # multiple formats + 'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070', + 'info_dict': { + 'id': '16070', + 'ext': 'mp4', + 'title': 'Madarasapatinam', + 'description': 'md5:63b9b8ed79189c6f0418c26d9a3452ca', + 'formats': 'mincount:2', + }, + 'params': { + 'skip_download': True, + } + }] def _real_extract(self, url): video_id = self._match_id(url) @@ -29,14 +43,28 @@ class BigflixIE(InfoExtractor): r']+class=["\']pagetitle["\'][^>]*>(.+?)', webpage, 'title') - video_url = b64decode(compat_urllib_parse_unquote(self._search_regex( - r'file=([^&]+)', webpage, 'video url')).encode('ascii')).decode('utf-8') + def decode_url(quoted_b64_url): + return base64.b64decode(compat_urllib_parse_unquote( + quoted_b64_url)).encode('ascii').decode('utf-8') + + formats = [{ + 'url': decode_url(encoded_url), + 'format_id': '%sp' % height, + 'height': int(height), + } for height, encoded_url in re.findall( + r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage)] + + if not formats: + formats.append({ + 'url': decode_url(self._search_regex( + r'file=([^&]+)', webpage, 'video url')), + }) description = self._html_search_meta('description', webpage) return { 'id': video_id, 'title': title, - 'url': video_url, 'description': description, + 'formats': formats } From a9bbd26f1d2bb45205f9fbd2626569522049e40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 10 Jan 2016 10:49:27 +0600 Subject: [PATCH 087/182] [bigflix] Improve formats extraction --- youtube_dl/extractor/bigflix.py | 41 ++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py index b7e498436..24720de86 100644 --- a/youtube_dl/extractor/bigflix.py +++ b/youtube_dl/extractor/bigflix.py @@ -20,7 +20,7 @@ class BigflixIE(InfoExtractor): 'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d', } }, { - # multiple formats + # 2 formats 'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070', 'info_dict': { 'id': '16070', @@ -32,6 +32,10 @@ class BigflixIE(InfoExtractor): 'params': { 'skip_download': True, } + }, { + # multiple formats + 'url': 'http://www.bigflix.com/Malayalam-movies/Drama-movies/Indian-Rupee/15967', + 'only_matching': True, }] def _real_extract(self, url): @@ -45,20 +49,31 @@ class BigflixIE(InfoExtractor): def decode_url(quoted_b64_url): return base64.b64decode(compat_urllib_parse_unquote( - quoted_b64_url)).encode('ascii').decode('utf-8') + quoted_b64_url).encode('ascii')).decode('utf-8') - formats = [{ - 'url': decode_url(encoded_url), - 'format_id': '%sp' % height, - 'height': int(height), - } for height, encoded_url in re.findall( - r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage)] + formats = [] + for height, encoded_url in re.findall( + r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage): + video_url = decode_url(encoded_url) + f = { + 'url': video_url, + 'format_id': '%sp' % height, + 'height': int(height), + } + if video_url.startswith('rtmp'): + f['ext'] = 'flv' + formats.append(f) - if not formats: - formats.append({ - 'url': decode_url(self._search_regex( - r'file=([^&]+)', webpage, 'video url')), - }) + file_url = self._search_regex( + r'file=([^&]+)', webpage, 'video url', default=None) + if file_url: + video_url = decode_url(file_url) + if all(f['url'] != video_url for f in formats): + formats.append({ + 'url': decode_url(file_url), + }) + + self._sort_formats(formats) description = self._html_search_meta('description', webpage) From 3fc088f8c7f61026943a62fc28e051a7d3a6bdd5 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 10 Jan 2016 07:45:41 +0100 Subject: [PATCH 088/182] [dcn] extract video ids in season entries --- youtube_dl/extractor/dcn.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py index 8f48571de..15a1c40f7 100644 --- a/youtube_dl/extractor/dcn.py +++ b/youtube_dl/extractor/dcn.py @@ -5,7 +5,10 @@ import re import base64 from .common import InfoExtractor -from ..compat import compat_urllib_parse +from ..compat import ( + compat_urllib_parse, + compat_str, +) from ..utils import ( int_or_none, parse_iso8601, @@ -186,7 +189,8 @@ class DCNSeasonIE(InfoExtractor): entries = [] for video in show['videos']: + video_id = compat_str(video['id']) entries.append(self.url_result( - 'http://www.dcndigital.ae/media/%s' % video['id'], 'DCNVideo')) + 'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo', video_id)) return self.playlist_result(entries, season_id, title) From 2334762b03dec5da4d6788539e3e11192eb97010 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 10 Jan 2016 07:55:58 +0100 Subject: [PATCH 089/182] [shahid] raise ExtractorError if the video is DRM protected --- youtube_dl/extractor/shahid.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py index f76fb12c0..1178b7a27 100644 --- a/youtube_dl/extractor/shahid.py +++ b/youtube_dl/extractor/shahid.py @@ -73,6 +73,9 @@ class ShahidIE(InfoExtractor): 'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-%s.html' % (video_id, api_vars['type']), video_id, 'Downloading player JSON') + if player.get('drm'): + raise ExtractorError('This video is DRM protected.', expected=True) + formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4') video = self._download_json( From 7e8a800f29d52cbc6057638595df05ac69f622bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 10 Jan 2016 14:26:27 +0100 Subject: [PATCH 090/182] [bigflix] Use correct indentation to make flake8 happy --- youtube_dl/extractor/bigflix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py index 24720de86..33762ad93 100644 --- a/youtube_dl/extractor/bigflix.py +++ b/youtube_dl/extractor/bigflix.py @@ -53,7 +53,7 @@ class BigflixIE(InfoExtractor): formats = [] for height, encoded_url in re.findall( - r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage): + r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage): video_url = decode_url(encoded_url) f = { 'url': video_url, From 3c91e41614a0f3bfab8710e861c895b78d93bce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 10 Jan 2016 14:32:53 +0100 Subject: [PATCH 091/182] [downloader/fragment] Don't fail if the 'Content-Length' header is missing In some dailymotion videos (like http://www.dailymotion.com/video/x3k0dtv from #8156) the segments URLs don't have the 'Content-Length' header and HttpFD sets the 'totat_bytes' field to None, so we also use '0' in that case (since we do different math operations with it). --- youtube_dl/downloader/fragment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py index 5a64b29ee..3da554622 100644 --- a/youtube_dl/downloader/fragment.py +++ b/youtube_dl/downloader/fragment.py @@ -66,7 +66,7 @@ class FragmentFD(FileDownloader): if s['status'] not in ('downloading', 'finished'): return - frag_total_bytes = s.get('total_bytes', 0) + frag_total_bytes = s.get('total_bytes') or 0 if s['status'] == 'finished': state['downloaded_bytes'] += frag_total_bytes state['frag_index'] += 1 From dfb1b1468cef4ddc7ecc43776abce03763f8e426 Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Sun, 10 Jan 2016 16:17:47 +0100 Subject: [PATCH 092/182] Fix typos Closes #8200. --- devscripts/gh-pages/update-copyright.py | 2 +- test/test_write_annotations.py | 2 +- youtube_dl/YoutubeDL.py | 2 +- youtube_dl/extractor/common.py | 4 ++-- youtube_dl/extractor/facebook.py | 4 ++-- youtube_dl/extractor/generic.py | 4 ++-- youtube_dl/extractor/ivi.py | 2 +- youtube_dl/extractor/mdr.py | 2 +- youtube_dl/extractor/nbc.py | 2 +- youtube_dl/extractor/nhl.py | 2 +- youtube_dl/extractor/ora.py | 2 +- youtube_dl/extractor/testurl.py | 2 +- youtube_dl/extractor/tv4.py | 2 +- youtube_dl/extractor/videomore.py | 2 +- youtube_dl/swfinterp.py | 2 +- youtube_dl/utils.py | 4 ++-- 16 files changed, 20 insertions(+), 20 deletions(-) diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py index 3663c8afe..e6c3abc8d 100755 --- a/devscripts/gh-pages/update-copyright.py +++ b/devscripts/gh-pages/update-copyright.py @@ -5,7 +5,7 @@ from __future__ import with_statement, unicode_literals import datetime import glob -import io # For Python 2 compatibilty +import io # For Python 2 compatibility import os import re diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py index 84b8f39e0..8de08f2d6 100644 --- a/test/test_write_annotations.py +++ b/test/test_write_annotations.py @@ -66,7 +66,7 @@ class TestAnnotations(unittest.TestCase): textTag = a.find('TEXT') text = textTag.text self.assertTrue(text in expected) # assertIn only added in python 2.7 - # remove the first occurance, there could be more than one annotation with the same text + # remove the first occurrence, there could be more than one annotation with the same text expected.remove(text) # We should have seen (and removed) all the expected annotation texts. self.assertEqual(len(expected), 0, 'Not all expected annotations were found.') diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 3b2be3159..d50b7cfed 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1312,7 +1312,7 @@ class YoutubeDL(object): # only set the 'formats' fields if the original info_dict list them # otherwise we end up with a circular reference, the first (and unique) # element in the 'formats' field in info_dict is info_dict itself, - # wich can't be exported to json + # which can't be exported to json info_dict['formats'] = formats if self.params.get('listformats'): self.list_formats(info_dict) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 0719c7bcd..b05b22a94 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -313,9 +313,9 @@ class InfoExtractor(object): except ExtractorError: raise except compat_http_client.IncompleteRead as e: - raise ExtractorError('A network error has occured.', cause=e, expected=True) + raise ExtractorError('A network error has occurred.', cause=e, expected=True) except (KeyError, StopIteration) as e: - raise ExtractorError('An extractor error has occured.', cause=e) + raise ExtractorError('An extractor error has occurred.', cause=e) def set_downloader(self, downloader): """Sets the downloader for this IE.""" diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 5e43f2359..ec699ba54 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -105,7 +105,7 @@ class FacebookIE(InfoExtractor): login_results, 'login error', default=None, group='error') if error: raise ExtractorError('Unable to login: %s' % error, expected=True) - self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') + self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.') return fb_dtsg = self._search_regex( @@ -126,7 +126,7 @@ class FacebookIE(InfoExtractor): check_response = self._download_webpage(check_req, None, note='Confirming login') if re.search(r'id="checkpointSubmitButton"', check_response) is not None: - self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.') + self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err)) return diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index d79e1adc9..b3f8efc80 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -487,7 +487,7 @@ class GenericIE(InfoExtractor): 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9', } }, - # Embeded Ustream video + # Embedded Ustream video { 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm', 'md5': '27b99cdb639c9b12a79bca876a073417', @@ -1644,7 +1644,7 @@ class GenericIE(InfoExtractor): if myvi_url: return self.url_result(myvi_url) - # Look for embeded soundcloud player + # Look for embedded soundcloud player mobj = re.search( r'https?://(?:w\.)?soundcloud\.com/player[^"]+)"', webpage) diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index d0f00cdea..472d72b4c 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -32,7 +32,7 @@ class IviIE(InfoExtractor): }, 'skip': 'Only works from Russia', }, - # Serial's serie + # Serial's series { 'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/9549', 'md5': '221f56b35e3ed815fde2df71032f4b3e', diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py index 88334889e..425fc9e2a 100644 --- a/youtube_dl/extractor/mdr.py +++ b/youtube_dl/extractor/mdr.py @@ -17,7 +17,7 @@ class MDRIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P\d+)(?:_.+?)?\.html' _TESTS = [{ - # MDR regularily deletes its videos + # MDR regularly deletes its videos 'url': 'http://www.mdr.de/fakt/video189002.html', 'only_matching': True, }, { diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py index 340c922bd..1dd54c2f1 100644 --- a/youtube_dl/extractor/nbc.py +++ b/youtube_dl/extractor/nbc.py @@ -100,7 +100,7 @@ class NBCSportsVPlayerIE(InfoExtractor): class NBCSportsIE(InfoExtractor): - # Does not include https becuase its certificate is invalid + # Does not include https because its certificate is invalid _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P[0-9a-z-]+)' _TEST = { diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index e98a5ef89..8d5ce46ad 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -223,7 +223,7 @@ class NHLVideocenterIE(NHLBaseInfoExtractor): response = self._download_webpage(request_url, playlist_title) response = self._fix_json(response) if not response.strip(): - self._downloader.report_warning('Got an empty reponse, trying ' + self._downloader.report_warning('Got an empty response, trying ' 'adding the "newvideos" parameter') response = self._download_webpage(request_url + '&newvideos=true', playlist_title) diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py index 9c4255a2d..02de1502a 100644 --- a/youtube_dl/extractor/ora.py +++ b/youtube_dl/extractor/ora.py @@ -37,7 +37,7 @@ class OraTVIE(InfoExtractor): formats = self._extract_m3u8_formats( m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False) - # simular to GameSpotIE + # similar to GameSpotIE m3u8_path = compat_urlparse.urlparse(m3u8_url).path QUALITIES_RE = r'((,[a-z]+\d+)+,?)' available_qualities = self._search_regex( diff --git a/youtube_dl/extractor/testurl.py b/youtube_dl/extractor/testurl.py index c7d559315..46918adb0 100644 --- a/youtube_dl/extractor/testurl.py +++ b/youtube_dl/extractor/testurl.py @@ -7,7 +7,7 @@ from ..utils import ExtractorError class TestURLIE(InfoExtractor): - """ Allows adressing of the test cases as test:yout.*be_1 """ + """ Allows addressing of the test cases as test:yout.*be_1 """ IE_DESC = False # Do not list _VALID_URL = r'test(?:url)?:(?P(?P.+?)(?:_(?P[0-9]+))?)$' diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py index 1c4b6d635..343edf206 100644 --- a/youtube_dl/extractor/tv4.py +++ b/youtube_dl/extractor/tv4.py @@ -67,7 +67,7 @@ class TV4IE(InfoExtractor): info = self._download_json( 'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON') - # If is_geo_restricted is true, it doesn't neceserally mean we can't download it + # If is_geo_restricted is true, it doesn't necessarily mean we can't download it if info['is_geo_restricted']: self.report_warning('This content might not be available in your country due to licensing restrictions.') if info['requires_subscription']: diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py index a66d6de23..fcee940e6 100644 --- a/youtube_dl/extractor/videomore.py +++ b/youtube_dl/extractor/videomore.py @@ -170,7 +170,7 @@ class VideomoreVideoIE(InfoExtractor): 'skip_download': True, }, }, { - # season single serie with og:video:iframe + # season single series with og:video:iframe 'url': 'http://videomore.ru/poslednii_ment/1_sezon/14_seriya', 'only_matching': True, }, { diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py index e60505ace..06c1d6cc1 100644 --- a/youtube_dl/swfinterp.py +++ b/youtube_dl/swfinterp.py @@ -689,7 +689,7 @@ class SWFInterpreter(object): elif mname in _builtin_classes: res = _builtin_classes[mname] else: - # Assume unitialized + # Assume uninitialized # TODO warn here res = undefined stack.append(res) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index da4ec7f20..9c1c0e0bd 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -984,7 +984,7 @@ def date_from_str(date_str): if sign == '-': time = -time unit = match.group('unit') - # A bad aproximation? + # A bad approximation? if unit == 'month': unit = 'day' time *= 30 @@ -1307,7 +1307,7 @@ def parse_filesize(s): if s is None: return None - # The lower-case forms are of course incorrect and inofficial, + # The lower-case forms are of course incorrect and unofficial, # but we support those too _UNIT_TABLE = { 'B': 1, From 36a0e46c39ea4f211dea9944177976e8f8364736 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 10 Jan 2016 17:55:41 +0100 Subject: [PATCH 093/182] fix typos --- youtube_dl/YoutubeDL.py | 4 ++-- youtube_dl/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d50b7cfed..6afc1b730 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -241,7 +241,7 @@ class YoutubeDL(object): - "detect_or_warn": check whether we can do anything about it, warn otherwise (default) source_address: (Experimental) Client-side IP address to bind to. - call_home: Boolean, true iff we are allowed to contact the + call_home: Boolean, true if we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. listformats: Print an overview of available video formats and exit. @@ -590,7 +590,7 @@ class YoutubeDL(object): return None def _match_entry(self, info_dict, incomplete): - """ Returns None iff the file should be downloaded """ + """ Returns None if the file should be downloaded """ video_title = info_dict.get('title', info_dict.get('id', 'video')) if 'title' in info_dict: diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 9c1c0e0bd..e583299c5 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1857,7 +1857,7 @@ def encode_data_uri(data, mime_type): def age_restricted(content_limit, age_limit): - """ Returns True iff the content should be blocked """ + """ Returns True if the content should be blocked """ if age_limit is None: # No limit set return False From 6ec6cb4e956abbd76a6ff42336821770a5fbbcc7 Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 10 Jan 2016 19:27:22 +0100 Subject: [PATCH 094/182] Revert "fix typos" This reverts commit 36a0e46c39ea4f211dea9944177976e8f8364736. --- youtube_dl/YoutubeDL.py | 4 ++-- youtube_dl/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6afc1b730..d50b7cfed 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -241,7 +241,7 @@ class YoutubeDL(object): - "detect_or_warn": check whether we can do anything about it, warn otherwise (default) source_address: (Experimental) Client-side IP address to bind to. - call_home: Boolean, true if we are allowed to contact the + call_home: Boolean, true iff we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. listformats: Print an overview of available video formats and exit. @@ -590,7 +590,7 @@ class YoutubeDL(object): return None def _match_entry(self, info_dict, incomplete): - """ Returns None if the file should be downloaded """ + """ Returns None iff the file should be downloaded """ video_title = info_dict.get('title', info_dict.get('id', 'video')) if 'title' in info_dict: diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index e583299c5..9c1c0e0bd 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1857,7 +1857,7 @@ def encode_data_uri(data, mime_type): def age_restricted(content_limit, age_limit): - """ Returns True if the content should be blocked """ + """ Returns True iff the content should be blocked """ if age_limit is None: # No limit set return False From 192b9a571cd50b0fd924e9ecb926b28633e8c19c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 11 Jan 2016 21:56:19 +0600 Subject: [PATCH 095/182] [videomega] Mark broken --- youtube_dl/extractor/videomega.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py index 87aca327b..5e2e7cbac 100644 --- a/youtube_dl/extractor/videomega.py +++ b/youtube_dl/extractor/videomega.py @@ -8,6 +8,7 @@ from ..utils import sanitized_Request class VideoMegaIE(InfoExtractor): + _WORKING = False _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P[A-Za-z0-9]+)' _TESTS = [{ 'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA', From b924bfad684ac7e9209e7165df3e13af6545596d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 11 Jan 2016 21:58:32 +0600 Subject: [PATCH 096/182] [videott] Mark broken --- youtube_dl/extractor/videott.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py index 591024ead..2cd36508a 100644 --- a/youtube_dl/extractor/videott.py +++ b/youtube_dl/extractor/videott.py @@ -11,6 +11,7 @@ from ..utils import ( class VideoTtIE(InfoExtractor): + _WORKING = False ID_NAME = 'video.tt' IE_DESC = 'video.tt - Your True Tube' _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P[\da-zA-Z]{9})' From be27283ef68e96d2461875a2bfb8c9ce962cfe61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 11 Jan 2016 22:00:17 +0600 Subject: [PATCH 097/182] [iprima] Mark broken --- youtube_dl/extractor/iprima.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py index 36baf3245..073777f34 100644 --- a/youtube_dl/extractor/iprima.py +++ b/youtube_dl/extractor/iprima.py @@ -14,6 +14,7 @@ from ..utils import ( class IPrimaIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P[^?#]+)' _TESTS = [{ From 9cb1a06b6c8fbeb6cfdbf0533ea60a6624fa1246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 12 Jan 2016 22:09:38 +0600 Subject: [PATCH 098/182] [downloader/fragment] Remove unused code and fix zero division error --- youtube_dl/downloader/fragment.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py index 3da554622..d236ac737 100644 --- a/youtube_dl/downloader/fragment.py +++ b/youtube_dl/downloader/fragment.py @@ -78,17 +78,10 @@ class FragmentFD(FileDownloader): state['total_bytes_estimate'] = estimated_size state['elapsed'] = time_now - start - if s['status'] == 'finished': - progress = self.calc_percent(state['frag_index'], total_frags) - else: - frag_downloaded_bytes = s['downloaded_bytes'] - frag_progress = self.calc_percent(frag_downloaded_bytes, - frag_total_bytes) - progress = self.calc_percent(state['frag_index'], total_frags) - progress += frag_progress / float(total_frags) - + if s['status'] != 'finished': state['eta'] = self.calc_eta( - start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes) + start, time_now, estimated_size, + state['downloaded_bytes'] + s['downloaded_bytes']) state['speed'] = s.get('speed') self._hook_progress(state) From 709185a2648c991cc143272dda67e4b42e89c03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 12 Jan 2016 23:18:38 +0600 Subject: [PATCH 099/182] [downloader/fragment] More smooth calculations `downloaded_bytes` is now updated on each fragment progress hook invocation --- youtube_dl/downloader/fragment.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py index d236ac737..b2597f1e5 100644 --- a/youtube_dl/downloader/fragment.py +++ b/youtube_dl/downloader/fragment.py @@ -58,6 +58,11 @@ class FragmentFD(FileDownloader): 'frag_count': total_frags, 'filename': ctx['filename'], 'tmpfilename': ctx['tmpfilename'], + # Total complete fragments downloaded so far in bytes + '_complete_frags_downloaded_bytes': 0, + # Amount of fragment's bytes downloaded by the time of the previous + # frag progress hook invocation + '_prev_frag_downloaded_bytes': 0, } start = time.time() ctx['started'] = start @@ -67,22 +72,27 @@ class FragmentFD(FileDownloader): return frag_total_bytes = s.get('total_bytes') or 0 - if s['status'] == 'finished': - state['downloaded_bytes'] += frag_total_bytes - state['frag_index'] += 1 estimated_size = ( - (state['downloaded_bytes'] + frag_total_bytes) / + (state['_complete_frags_downloaded_bytes'] + frag_total_bytes) / (state['frag_index'] + 1) * total_frags) time_now = time.time() state['total_bytes_estimate'] = estimated_size state['elapsed'] = time_now - start - if s['status'] != 'finished': + if s['status'] == 'finished': + state['frag_index'] += 1 + state['downloaded_bytes'] += frag_total_bytes - state['_prev_frag_downloaded_bytes'] + state['_complete_frags_downloaded_bytes'] = state['downloaded_bytes'] + state['_prev_frag_downloaded_bytes'] = 0 + else: + frag_downloaded_bytes = s['downloaded_bytes'] + state['downloaded_bytes'] += frag_downloaded_bytes - state['_prev_frag_downloaded_bytes'] state['eta'] = self.calc_eta( start, time_now, estimated_size, - state['downloaded_bytes'] + s['downloaded_bytes']) + state['downloaded_bytes']) state['speed'] = s.get('speed') + state['_prev_frag_downloaded_bytes'] = frag_downloaded_bytes self._hook_progress(state) ctx['dl'].add_progress_hook(frag_progress_hook) From 16a348475cb7ab90e80c31a01eec1280f4fc4fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 12 Jan 2016 23:23:39 +0600 Subject: [PATCH 100/182] [dailymotion] Prefer direct links (Closes #8156) --- youtube_dl/extractor/dailymotion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 439fd42e8..b687ec4d6 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -149,14 +149,15 @@ class DailymotionIE(DailymotionBaseInfoExtractor): ext = determine_ext(media_url) if type_ == 'application/x-mpegURL' or ext == 'm3u8': formats.extend(self._extract_m3u8_formats( - media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)) + media_url, video_id, 'mp4', preference=-1, + m3u8_id='hls', fatal=False)) elif type_ == 'application/f4m' or ext == 'f4m': formats.extend(self._extract_f4m_formats( media_url, video_id, preference=-1, f4m_id='hds', fatal=False)) else: f = { 'url': media_url, - 'format_id': quality, + 'format_id': 'http-%s' % quality, } m = re.search(r'H264-(?P\d+)x(?P\d+)', media_url) if m: From b83b782dc4afeabbd356a329fd302b4410afa626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 13 Jan 2016 00:00:31 +0600 Subject: [PATCH 101/182] [downloader/fragment] Move helper data to context dict --- youtube_dl/downloader/fragment.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py index b2597f1e5..0c9113d0f 100644 --- a/youtube_dl/downloader/fragment.py +++ b/youtube_dl/downloader/fragment.py @@ -58,14 +58,17 @@ class FragmentFD(FileDownloader): 'frag_count': total_frags, 'filename': ctx['filename'], 'tmpfilename': ctx['tmpfilename'], + } + + start = time.time() + ctx.update({ + 'started': start, # Total complete fragments downloaded so far in bytes - '_complete_frags_downloaded_bytes': 0, + 'complete_frags_downloaded_bytes': 0, # Amount of fragment's bytes downloaded by the time of the previous # frag progress hook invocation - '_prev_frag_downloaded_bytes': 0, - } - start = time.time() - ctx['started'] = start + 'prev_frag_downloaded_bytes': 0, + }) def frag_progress_hook(s): if s['status'] not in ('downloading', 'finished'): @@ -74,7 +77,7 @@ class FragmentFD(FileDownloader): frag_total_bytes = s.get('total_bytes') or 0 estimated_size = ( - (state['_complete_frags_downloaded_bytes'] + frag_total_bytes) / + (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) / (state['frag_index'] + 1) * total_frags) time_now = time.time() state['total_bytes_estimate'] = estimated_size @@ -82,17 +85,17 @@ class FragmentFD(FileDownloader): if s['status'] == 'finished': state['frag_index'] += 1 - state['downloaded_bytes'] += frag_total_bytes - state['_prev_frag_downloaded_bytes'] - state['_complete_frags_downloaded_bytes'] = state['downloaded_bytes'] - state['_prev_frag_downloaded_bytes'] = 0 + state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes'] + ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes'] + ctx['prev_frag_downloaded_bytes'] = 0 else: frag_downloaded_bytes = s['downloaded_bytes'] - state['downloaded_bytes'] += frag_downloaded_bytes - state['_prev_frag_downloaded_bytes'] + state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes'] state['eta'] = self.calc_eta( start, time_now, estimated_size, state['downloaded_bytes']) state['speed'] = s.get('speed') - state['_prev_frag_downloaded_bytes'] = frag_downloaded_bytes + ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes self._hook_progress(state) ctx['dl'].add_progress_hook(frag_progress_hook) From bc0550c262db0bc756bb7e08fb945f3285fb196e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 13 Jan 2016 08:18:37 +0600 Subject: [PATCH 102/182] [pluralsight] Fix new player (Closes #8215) --- youtube_dl/extractor/pluralsight.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py index 55c11b3bf..12e1c2862 100644 --- a/youtube_dl/extractor/pluralsight.py +++ b/youtube_dl/extractor/pluralsight.py @@ -232,7 +232,7 @@ class PluralsightIE(PluralsightBaseIE): # { a = author, cn = clip_id, lc = end, m = name } return { - 'id': clip['clipName'], + 'id': clip.get('clipName') or clip['name'], 'title': '%s - %s' % (module['title'], clip['title']), 'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')), 'creator': author, From cc28492d31556b55bbd8fc574bf5d890305b22d2 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 13 Jan 2016 17:05:38 +0800 Subject: [PATCH 103/182] [youtube] Fix acodec and vcodec order In RFC6381, there's no rule stating that the first part of codecs should be video and the second part should be audio, while it seems the case for data reported by YouTube. --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 897641e42..e4f227f19 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1487,7 +1487,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if codecs: codecs = codecs.split(',') if len(codecs) == 2: - acodec, vcodec = codecs[0], codecs[1] + acodec, vcodec = codecs[1], codecs[0] else: acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0]) dct.update({ From 40cf7fcbd2e30747065ca7b8bf4467a9582a4aa9 Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 13 Jan 2016 13:29:00 +0100 Subject: [PATCH 104/182] [tudou] Add support for Albums and Playlists and extract more metadata --- youtube_dl/extractor/__init__.py | 6 +- youtube_dl/extractor/tudou.py | 94 ++++++++++++++++++++++++++------ 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 40c42d4d2..e4ae9332d 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -723,7 +723,11 @@ from .trilulilu import TriluliluIE from .trutube import TruTubeIE from .tube8 import Tube8IE from .tubitv import TubiTvIE -from .tudou import TudouIE +from .tudou import ( + TudouIE, + TudouPlaylistIE, + TudouAlbumIE, +) from .tumblr import TumblrIE from .tunein import ( TuneInClipIE, diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index 5f7ac4b35..da3cd76f7 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -4,10 +4,16 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..compat import compat_str +from ..utils import ( + int_or_none, + float_or_none, + unescapeHTML, +) class TudouIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/([^/]+/)*(?P[^/?#]+?)(?:\.html)?/?(?:$|[?#])' + IE_NAME = 'tudou' + _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:(?:programs|wlplay)/view|(?:listplay|albumplay)/[\w-]{11})/(?P[\w-]{11})' _TESTS = [{ 'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html', 'md5': '140a49ed444bd22f93330985d8475fcb', @@ -16,6 +22,11 @@ class TudouIE(InfoExtractor): 'ext': 'f4v', 'title': '卡马乔国足开大脚长传冲吊集锦', 'thumbnail': 're:^https?://.*\.jpg$', + 'timestamp': 1372113489000, + 'description': '卡马乔卡家军，开大脚先进战术不完全集锦！', + 'duration': 289.04, + 'view_count': int, + 'filesize': int, } }, { 'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/', @@ -24,10 +35,12 @@ class TudouIE(InfoExtractor): 'ext': 'f4v', 'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012', 'thumbnail': 're:^https?://.*\.jpg$', + 'timestamp': 1349207518000, + 'description': 'md5:294612423894260f2dcd5c6c04fe248b', + 'duration': 5478.33, + 'view_count': int, + 'filesize': int, } - }, { - 'url': 'http://www.tudou.com/albumplay/cJAHGih4yYg.html', - 'only_matching': True, }] _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf' @@ -42,24 +55,20 @@ class TudouIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + item_data = self._download_json( + 'http://www.tudou.com/tvp/getItemInfo.action?ic=%s' % video_id, video_id) - youku_vcode = self._search_regex( - r'vcode\s*:\s*[\'"]([^\'"]*)[\'"]', webpage, 'youku vcode', default=None) + youku_vcode = item_data.get('vcode') if youku_vcode: return self.url_result('youku:' + youku_vcode, ie='Youku') - title = self._search_regex( - r',kw\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'title') - thumbnail_url = self._search_regex( - r',pic\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'thumbnail URL', fatal=False) + title = unescapeHTML(item_data['kw']) + description = item_data.get('desc') + thumbnail_url = item_data.get('pic') + view_count = int_or_none(item_data.get('playTimes')) + timestamp = int_or_none(item_data.get('pt')) - player_url = self._search_regex( - r'playerUrl\s*:\s*[\'"]([^\'"]+\.swf)[\'"]', - webpage, 'player URL', default=self._PLAYER_URL) - - segments = self._parse_json(self._search_regex( - r'segs: \'([^\']+)\'', webpage, 'segments'), video_id) + segments = self._parse_json(item_data['itemSegs'], video_id) # It looks like the keys are the arguments that have to be passed as # the hd field in the request url, we pick the higher # Also, filter non-number qualities (see issue #3643). @@ -80,8 +89,13 @@ class TudouIE(InfoExtractor): 'ext': ext, 'title': title, 'thumbnail': thumbnail_url, + 'description': description, + 'view_count': view_count, + 'timestamp': timestamp, + 'duration': float_or_none(part.get('seconds'), 1000), + 'filesize': int_or_none(part.get('size')), 'http_headers': { - 'Referer': player_url, + 'Referer': self._PLAYER_URL, }, } result.append(part_info) @@ -92,3 +106,47 @@ class TudouIE(InfoExtractor): 'id': video_id, 'title': title, } + + +class TudouPlaylistIE(InfoExtractor): + IE_NAME = 'tudou:playlist' + _VALID_URL = r'https?://(?:www\.)?tudou\.com/listplay/(?P[\w-]{11})\.html' + _TESTS = [{ + 'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo.html', + 'info_dict': { + 'id': 'zzdE77v6Mmo', + }, + 'playlist_mincount': 209, + }] + + def _real_extract(self, url): + playlist_id = self._match_id(url) + playlist_data = self._download_json( + 'http://www.tudou.com/tvp/plist.action?lcode=%s' % playlist_id, playlist_id) + entries = [self.url_result( + 'http://www.tudou.com/programs/view/%s' % item['icode'], + 'Tudou', item['icode'], + item['kw']) for item in playlist_data['items']] + return self.playlist_result(entries, playlist_id) + + +class TudouAlbumIE(InfoExtractor): + IE_NAME = 'tudou:album' + _VALID_URL = r'https?://(?:www\.)?tudou\.com/album(?:cover|play)/(?P[\w-]{11})' + _TESTS = [{ + 'url': 'http://www.tudou.com/albumplay/v5qckFJvNJg.html', + 'info_dict': { + 'id': 'v5qckFJvNJg', + }, + 'playlist_mincount': 45, + }] + + def _real_extract(self, url): + album_id = self._match_id(url) + album_data = self._download_json( + 'http://www.tudou.com/tvp/alist.action?acode=%s' % album_id, album_id) + entries = [self.url_result( + 'http://www.tudou.com/programs/view/%s' % item['icode'], + 'Tudou', item['icode'], + item['kw']) for item in album_data['items']] + return self.playlist_result(entries, album_id) From e37afbe0b8a1222cb214ad0bec9a53bb7953531d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 14 Jan 2016 00:16:23 +0100 Subject: [PATCH 105/182] [YoutubeDL] urlopen: disable the 'file:' protocol (#8227) If someone is running youtube-dl on a server to deliver files, the user could input 'file:///some/important/file' and youtube-dl would save that file as a video giving access to sensitive information to the user. 'file:' urls can be filtered, but the user can use an URL to a crafted m3u8 manifest like: #EXTM3U #EXT-X-MEDIA-SEQUENCE:0 #EXTINF:10.0 file:///etc/passwd #EXT-X-ENDLIST With this patch 'file:' URLs raise URLError like for unknown protocols. --- test/test_YoutubeDL.py | 7 ++++++- youtube_dl/YoutubeDL.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 0388c0bf3..0caa43843 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -12,7 +12,7 @@ import copy from test.helper import FakeYDL, assertRegexpMatches from youtube_dl import YoutubeDL -from youtube_dl.compat import compat_str +from youtube_dl.compat import compat_str, compat_urllib_error from youtube_dl.extractor import YoutubeIE from youtube_dl.postprocessor.common import PostProcessor from youtube_dl.utils import ExtractorError, match_filter_func @@ -631,6 +631,11 @@ class TestYoutubeDL(unittest.TestCase): result = get_ids({'playlist_items': '10'}) self.assertEqual(result, []) + def test_urlopen_no_file_protocol(self): + # see https://github.com/rg3/youtube-dl/issues/8227 + ydl = YDL() + self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d50b7cfed..e8ce58604 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1986,8 +1986,14 @@ class YoutubeDL(object): https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel) ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel) data_handler = compat_urllib_request_DataHandler() - opener = compat_urllib_request.build_opener( - proxy_handler, https_handler, cookie_processor, ydlh, data_handler) + unknown_handler = compat_urllib_request.UnknownHandler() + handlers = (proxy_handler, https_handler, cookie_processor, ydlh, data_handler, unknown_handler) + # we don't use build_opener because it automatically adds FileHandler, + # which can be used for malicious purposes (see + # https://github.com/rg3/youtube-dl/issues/8227) + opener = compat_urllib_request.OpenerDirector() + for handler in handlers: + opener.add_handler(handler) # Delete the default user-agent header, which would otherwise apply in # cases where our custom HTTP handler doesn't come into play From 6240b0a278781a3b584a9dd6d57191b2472c0fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 14 Jan 2016 08:14:01 +0100 Subject: [PATCH 106/182] [YoutubeDL] urlopen: use build_opener again Otherwise we would need to manually add handlers like HTTPRedirectHandler, instead we add a customized FileHandler instance that raises an error. --- youtube_dl/YoutubeDL.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e8ce58604..ccad5f2ea 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1986,14 +1986,19 @@ class YoutubeDL(object): https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel) ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel) data_handler = compat_urllib_request_DataHandler() - unknown_handler = compat_urllib_request.UnknownHandler() - handlers = (proxy_handler, https_handler, cookie_processor, ydlh, data_handler, unknown_handler) - # we don't use build_opener because it automatically adds FileHandler, - # which can be used for malicious purposes (see + + # When passing our own FileHandler instance, build_opener won't add the + # default FileHandler and allows us to disable the file protocol, which + # can be used for malicious purposes (see # https://github.com/rg3/youtube-dl/issues/8227) - opener = compat_urllib_request.OpenerDirector() - for handler in handlers: - opener.add_handler(handler) + file_handler = compat_urllib_request.FileHandler() + + def file_open(*args, **kwargs): + raise compat_urllib_error.URLError('file protocol is disabled') + file_handler.file_open = file_open + + opener = compat_urllib_request.build_opener( + proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler) # Delete the default user-agent header, which would otherwise apply in # cases where our custom HTTP handler doesn't come into play From 4240d504963bb6d1c7bd7c288a7874f9d8dc042b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 14 Jan 2016 14:07:54 +0100 Subject: [PATCH 107/182] [YoutubeDL] improve error message for file:/// URLs --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index ccad5f2ea..4915fbd45 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1994,7 +1994,7 @@ class YoutubeDL(object): file_handler = compat_urllib_request.FileHandler() def file_open(*args, **kwargs): - raise compat_urllib_error.URLError('file protocol is disabled') + raise compat_urllib_error.URLError('file:/// protocol is explicitly disabled in youtube-dl for security reasons') file_handler.file_open = file_open opener = compat_urllib_request.build_opener( From 4511c1976d0a06394a000333a020a4d3668072fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 19:57:20 +0600 Subject: [PATCH 108/182] [beeg] Fix extraction (Closes #8225) --- youtube_dl/extractor/beeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py index c8d921daf..d0174b818 100644 --- a/youtube_dl/extractor/beeg.py +++ b/youtube_dl/extractor/beeg.py @@ -60,7 +60,7 @@ class BeegIE(InfoExtractor): def decrypt_url(encrypted_url): encrypted_url = self._proto_relative_url( - encrypted_url.replace('{DATA_MARKERS}', ''), 'http:') + encrypted_url.replace('{DATA_MARKERS}', ''), 'https:') key = self._search_regex( r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None) if not key: From abb893e6e45b0b0c6ec0e3a1d29dbd1746cbee96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 19:57:56 +0600 Subject: [PATCH 109/182] [beeg] Update API URL --- youtube_dl/extractor/beeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py index d0174b818..34c2a756f 100644 --- a/youtube_dl/extractor/beeg.py +++ b/youtube_dl/extractor/beeg.py @@ -34,7 +34,7 @@ class BeegIE(InfoExtractor): video_id = self._match_id(url) video = self._download_json( - 'http://beeg.com/api/v5/video/%s' % video_id, video_id) + 'https://api.beeg.com/api/v5/video/%s' % video_id, video_id) def split(o, e): def cut(s, x): From 11c60089a8772a2d12288f0ff382866e516f9a4b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 14 Jan 2016 15:43:21 +0100 Subject: [PATCH 110/182] release 2016.01.14 --- docs/supportedsites.md | 11 +++++++---- youtube_dl/version.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 8d0c7b97a..eb160bd2f 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -65,6 +65,7 @@ - **Beeg** - **BehindKink** - **Bet** + - **Bigflix** - **Bild**: Bild.de - **BiliBili** - **BleacherReport** @@ -251,7 +252,7 @@ - **Instagram** - **instagram:user**: Instagram user profile - **InternetVideoArchive** - - **IPrima** + - **IPrima** (Currently broken) - **iqiyi**: 爱奇艺 - **Ir90Tv** - **ivi**: ivi.ru @@ -602,7 +603,9 @@ - **TruTube** - **Tube8** - **TubiTv** - - **Tudou** + - **tudou** + - **tudou:album** + - **tudou:playlist** - **Tumblr** - **tunein:clip** - **tunein:program** @@ -655,12 +658,12 @@ - **video.mit.edu** - **VideoDetective** - **videofy.me** - - **VideoMega** + - **VideoMega** (Currently broken) - **videomore** - **videomore:season** - **videomore:video** - **VideoPremium** - - **VideoTt**: video.tt - Your True Tube + - **VideoTt**: video.tt - Your True Tube (Currently broken) - **videoweed**: VideoWeed - **Vidme** - **Vidzi** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 7030903c0..4d433b667 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2016.01.09' +__version__ = '2016.01.14' From 30e2f2d76f6dd52803effce14fa14f3a8051c84a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 14 Jan 2016 16:28:46 +0100 Subject: [PATCH 111/182] [YoutubeDL] use a more correct terminology in the error message for file:// URLs --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4915fbd45..6b73b8e06 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1994,7 +1994,7 @@ class YoutubeDL(object): file_handler = compat_urllib_request.FileHandler() def file_open(*args, **kwargs): - raise compat_urllib_error.URLError('file:/// protocol is explicitly disabled in youtube-dl for security reasons') + raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons') file_handler.file_open = file_open opener = compat_urllib_request.build_opener( From fbd90643cb123011a224da58b4ff1c4ba1c4f8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 21:48:08 +0600 Subject: [PATCH 112/182] [vodlocker] Fix extraction (Closes #8231) --- youtube_dl/extractor/vodlocker.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py index 357594a11..a97995a6d 100644 --- a/youtube_dl/extractor/vodlocker.py +++ b/youtube_dl/extractor/vodlocker.py @@ -5,12 +5,13 @@ from .common import InfoExtractor from ..compat import compat_urllib_parse from ..utils import ( ExtractorError, + NO_DEFAULT, sanitized_Request, ) class VodlockerIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?:embed-)?(?P[0-9a-zA-Z]+)(?:\..*?)?' + _VALID_URL = r'https?://(?:www\.)?vodlocker\.(?:com|city)/(?:embed-)?(?P[0-9a-zA-Z]+)(?:\..*?)?' _TESTS = [{ 'url': 'http://vodlocker.com/e8wvyzz4sl42', @@ -43,16 +44,31 @@ class VodlockerIE(InfoExtractor): webpage = self._download_webpage( req, video_id, 'Downloading video page') + def extract_file_url(html, default=NO_DEFAULT): + return self._search_regex( + r'file:\s*"(http[^\"]+)",', html, 'file url', default=default) + + video_url = extract_file_url(webpage, default=None) + + if not video_url: + embed_url = self._search_regex( + r']+src=(["\'])(?P(?:https?://)?vodlocker\.(?:com|city)/embed-.+?)\1', + webpage, 'embed url', group='url') + embed_webpage = self._download_webpage( + embed_url, video_id, 'Downloading embed webpage') + video_url = extract_file_url(embed_webpage) + thumbnail_webpage = embed_webpage + else: + thumbnail_webpage = webpage + title = self._search_regex( r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title') thumbnail = self._search_regex( - r'image:\s*"(http[^\"]+)",', webpage, 'thumbnail') - url = self._search_regex( - r'file:\s*"(http[^\"]+)",', webpage, 'file url') + r'image:\s*"(http[^\"]+)",', thumbnail_webpage, 'thumbnail', fatal=False) formats = [{ 'format_id': 'sd', - 'url': url, + 'url': video_url, }] return { From 5cc9c5dfa8f731b6582b092e06f78cccbaefc3c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 21:53:24 +0600 Subject: [PATCH 113/182] [unistra] Fix extraction --- youtube_dl/extractor/unistra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py index f70978299..594bee4f9 100644 --- a/youtube_dl/extractor/unistra.py +++ b/youtube_dl/extractor/unistra.py @@ -38,7 +38,7 @@ class UnistraIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - files = set(re.findall(r'file\s*:\s*"([^"]+)"', webpage)) + files = set(re.findall(r'file\s*:\s*"(/[^"]+)"', webpage)) quality = qualities(['SD', 'HD']) formats = [] From 163e8369b0d2f6b8cc59dd1e93b20a980590648f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 22:05:04 +0600 Subject: [PATCH 114/182] [ntvde] Fix extraction --- youtube_dl/extractor/ntvde.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py index d2cfe0961..8268eace7 100644 --- a/youtube_dl/extractor/ntvde.py +++ b/youtube_dl/extractor/ntvde.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from .common import InfoExtractor +from ..compat import compat_urlparse from ..utils import ( int_or_none, js_to_json, @@ -42,18 +43,24 @@ class NTVDeIE(InfoExtractor): webpage, 'player data'), video_id, transform_source=js_to_json) duration = parse_duration(vdata.get('duration')) - formats = [{ - 'format_id': 'flash', - 'url': 'rtmp://fms.n-tv.de/' + vdata['video'], - }, { - 'format_id': 'mobile', - 'url': 'http://video.n-tv.de' + vdata['videoMp4'], - 'tbr': 400, # estimation - }] - m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8'] - formats.extend(self._extract_m3u8_formats( - m3u8_url, video_id, ext='mp4', - entry_protocol='m3u8_native', preference=0)) + + formats = [] + if vdata.get('video'): + formats.append({ + 'format_id': 'flash', + 'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'], + }) + if vdata.get('videoMp4'): + formats.append({ + 'format_id': 'mobile', + 'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']), + 'tbr': 400, # estimation + }) + if vdata.get('videoM3u8'): + m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8']) + formats.extend(self._extract_m3u8_formats( + m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native', + preference=0, m3u8_id='hls', fatal=False)) self._sort_formats(formats) return { From 4654c1d01613e26d782c95b13ce60e5fdd84892a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 22:07:42 +0600 Subject: [PATCH 115/182] [orf:fm4] Extend _VALID_URL (Closes #8234) --- youtube_dl/extractor/orf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index 2e6c9872b..da598e7f7 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -170,7 +170,7 @@ class ORFOE1IE(InfoExtractor): class ORFFM4IE(InfoExtractor): IE_NAME = 'orf:fm4' IE_DESC = 'radio FM4' - _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P[0-9]+)/(?P\w+)' + _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P[0-9]+)/(?P\w+)' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) From 986986064ec102b0d97b4ab008ae38ede6358796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 22:11:33 +0600 Subject: [PATCH 116/182] [orf:fm4] Add test --- youtube_dl/extractor/orf.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index da598e7f7..c54775d54 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -172,6 +172,20 @@ class ORFFM4IE(InfoExtractor): IE_DESC = 'radio FM4' _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P[0-9]+)/(?P\w+)' + _TEST = { + 'url': 'http://fm4.orf.at/player/20160110/IS/', + 'md5': '01e736e8f1cef7e13246e880a59ad298', + 'info_dict': { + 'id': '2016-01-10_2100_tl_54_7DaysSun13_11244', + 'ext': 'mp3', + 'title': 'Im Sumpf', + 'description': 'md5:384c543f866c4e422a55f66a62d669cd', + 'duration': 7173, + 'timestamp': 1452456073, + 'upload_date': '20160110', + }, + } + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) show_date = mobj.group('date') From 6b559c2fbcf70158bd84b3b5892ecd5fc4b03e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 14 Jan 2016 22:12:24 +0600 Subject: [PATCH 117/182] [ntvde] Improve regex --- youtube_dl/extractor/ntvde.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py index 8268eace7..a83e85cb8 100644 --- a/youtube_dl/extractor/ntvde.py +++ b/youtube_dl/extractor/ntvde.py @@ -35,7 +35,7 @@ class NTVDeIE(InfoExtractor): webpage = self._download_webpage(url, video_id) info = self._parse_json(self._search_regex( - r'(?s)ntv.pageInfo.article =\s(\{.*?\});', webpage, 'info'), + r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'), video_id, transform_source=js_to_json) timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp')) vdata = self._parse_json(self._search_regex( From 0baedd1851692a4b9f94c08b3eae5d57acf07f09 Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 13 Jan 2016 16:11:49 +0100 Subject: [PATCH 118/182] [prosiebensat1] add support for 7tv.de --- youtube_dl/extractor/prosiebensat1.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index baa54a3af..953df3efc 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -20,7 +20,7 @@ from ..utils import ( class ProSiebenSat1IE(InfoExtractor): IE_NAME = 'prosiebensat1' IE_DESC = 'ProSiebenSat.1 Digital' - _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P.+)' + _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P.+)' _TESTS = [ { @@ -172,6 +172,20 @@ class ProSiebenSat1IE(InfoExtractor): }, 'playlist_count': 2, }, + { + 'url': 'http://www.7tv.de/circus-halligalli/615-best-of-circus-halligalli-ganze-folge', + 'info_dict': { + 'id': '4187506', + 'ext': 'flv', + 'title': 'Best of Circus HalliGalli', + 'description': 'md5:8849752efd90b9772c9db6fdf87fb9e9', + 'upload_date': '20151229', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + }, ] _CLIPID_REGEXES = [ @@ -186,12 +200,14 @@ class ProSiebenSat1IE(InfoExtractor): r'\s*

(.+?)

', r'

\s*(.+?)

', r'
\s*

([^<]+)

\s*
', + r'

\s*(.+?)

', ] _DESCRIPTION_REGEXES = [ r'

\s*(.+?)

', r'
\s*

Beschreibung: (.+?)

', r'
\s*
\s*\s*(.+?)\s*