From f77970765adb0ffe837706ef8eb041ea448c34b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 20 Apr 2016 03:37:23 +0600 Subject: [PATCH 01/26] [people] Add extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/people.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 youtube_dl/extractor/people.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index aa83daa02..9b8e1ab25 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -559,6 +559,7 @@ from .pandoratv import PandoraTVIE from .parliamentliveuk import ParliamentLiveUKIE from .patreon import PatreonIE from .pbs import PBSIE +from .people import PeopleIE from .periscope import PeriscopeIE from .philharmoniedeparis import PhilharmonieDeParisIE from .phoenix import PhoenixIE diff --git a/youtube_dl/extractor/people.py b/youtube_dl/extractor/people.py new file mode 100644 index 000000000..f25459045 --- /dev/null +++ b/youtube_dl/extractor/people.py @@ -0,0 +1,33 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class PeopleIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?people\.com/people/videos/0,,(?P\d+),00\.html' + + _TEST = { + 'url': 'http://www.people.com/people/videos/0,,20995451,00.html', + 'info_dict': { + 'id': 'ref:20995451', + 'ext': 'mp4', + 'title': 'Astronaut Love Triangle Victim Speaks Out: “The Crime in 2007 Hasn’t Defined Us”', + 'description': 'Colleen Shipman speaks to PEOPLE for the first time about life after the attack', + 'thumbnail': 're:^https?://.*\.jpg', + 'duration': 246.318, + 'timestamp': 1458720585, + 'upload_date': '20160323', + 'uploader_id': '416418724', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + 'add_ie': ['BrightcoveNew'], + } + + def _real_extract(self, url): + return self.url_result( + 'http://players.brightcove.net/416418724/default_default/index.html?videoId=ref:%s' + % self._match_id(url), 'BrightcoveNew') From 107701fcfced36af62465f4a56a0a9723f5253bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 20 Apr 2016 03:40:02 +0600 Subject: [PATCH 02/26] [people] Remove bogus comment --- youtube_dl/extractor/people.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/people.py b/youtube_dl/extractor/people.py index f25459045..9ecdbc13b 100644 --- a/youtube_dl/extractor/people.py +++ b/youtube_dl/extractor/people.py @@ -21,7 +21,6 @@ class PeopleIE(InfoExtractor): 'uploader_id': '416418724', }, 'params': { - # m3u8 download 'skip_download': True, }, 'add_ie': ['BrightcoveNew'], From 494ab6db7394b39126d775efe03eb6dab428eff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 20 Apr 2016 22:14:32 +0600 Subject: [PATCH 03/26] [youtube] Capture and output login error message --- youtube_dl/extractor/youtube.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index a4dd628a1..44f98d294 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -125,6 +125,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor): if login_results is False: return False + error_msg = self._html_search_regex( + r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<', + login_results, 'error message', default=None) + if error_msg: + raise ExtractorError('Unable to login: %s' % error_msg, expected=True) + if re.search(r'id="errormsg_0_Passwd"', login_results) is not None: raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True) From 16e633a5d715a36fa4dbff1dbc8bfbf01ad081db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 20 Apr 2016 23:29:02 +0600 Subject: [PATCH 04/26] [quickvid] Remove extractor (Closes #9258) --- youtube_dl/extractor/extractors.py | 1 - youtube_dl/extractor/quickvid.py | 54 ------------------------------ 2 files changed, 55 deletions(-) delete mode 100644 youtube_dl/extractor/quickvid.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 9b8e1ab25..1f3172119 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -601,7 +601,6 @@ from .qqmusic import ( QQMusicToplistIE, QQMusicPlaylistIE, ) -from .quickvid import QuickVidIE from .r7 import R7IE from .radiode import RadioDeIE from .radiojavan import RadioJavanIE diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py deleted file mode 100644 index f414e2384..000000000 --- a/youtube_dl/extractor/quickvid.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..compat import ( - compat_urlparse, -) -from ..utils import ( - determine_ext, - int_or_none, -) - - -class QuickVidIE(InfoExtractor): - _VALID_URL = r'https?://(www\.)?quickvid\.org/watch\.php\?v=(?P[a-zA-Z_0-9-]+)' - _TEST = { - 'url': 'http://quickvid.org/watch.php?v=sUQT3RCG8dx', - 'md5': 'c0c72dd473f260c06c808a05d19acdc5', - 'info_dict': { - 'id': 'sUQT3RCG8dx', - 'ext': 'mp4', - 'title': 'Nick Offerman\'s Summer Reading Recap', - 'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$', - 'view_count': int, - }, - 'skip': 'Not accessible from Travis CI server', - } - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - title = self._html_search_regex(r'

(.*?)

', webpage, 'title') - view_count = int_or_none(self._html_search_regex( - r'(?s)
(.*?)
', - webpage, 'view count', fatal=False)) - video_code = self._search_regex( - r'(?s)', webpage, 'video code') - formats = [ - { - 'url': compat_urlparse.urljoin(url, src), - 'format_id': determine_ext(src, None), - } for src in re.findall(' Date: Wed, 20 Apr 2016 23:52:41 +0600 Subject: [PATCH 05/26] [tvigle] Fix extraction (Closes #9259) --- youtube_dl/extractor/tvigle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py index dc3a8334a..f3aba3530 100644 --- a/youtube_dl/extractor/tvigle.py +++ b/youtube_dl/extractor/tvigle.py @@ -58,7 +58,9 @@ class TvigleIE(InfoExtractor): if not video_id: webpage = self._download_webpage(url, display_id) video_id = self._html_search_regex( - r'class="video-preview current_playing" id="(\d+)">', + (r']+class=["\']player["\'][^>]+id=["\'](\d+)', + r'var\s+cloudId\s*=\s*["\'](\d+)', + r'class="video-preview current_playing" id="(\d+)"'), webpage, 'video id') video_data = self._download_json( From 2c0d9c6217f994b2742ff4e8e3fba7c7469b9f81 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 21 Apr 2016 13:06:06 +0800 Subject: [PATCH 06/26] [extractor/common] Allow empty post data --- youtube_dl/extractor/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 5269059d0..02cd2c003 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -382,7 +382,7 @@ class InfoExtractor(object): else: if query: url_or_request = update_url_query(url_or_request, query) - if data or headers: + if data is not None or headers: url_or_request = sanitized_Request(url_or_request, data, headers) try: return self._downloader.urlopen(url_or_request) From 4dccea8ad01d51dcdfd4ecc79442bbd30e29871e Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 21 Apr 2016 13:07:53 +0800 Subject: [PATCH 07/26] [streetvoice] Fix extraction The old API results in URLs with HTTP 403 from time to time. Hopefully fixes #9219. --- youtube_dl/extractor/streetvoice.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py index 6a57fa60a..e529051d1 100644 --- a/youtube_dl/extractor/streetvoice.py +++ b/youtube_dl/extractor/streetvoice.py @@ -14,7 +14,6 @@ class StreetVoiceIE(InfoExtractor): 'info_dict': { 'id': '94440', 'ext': 'mp3', - 'filesize': 4167053, 'title': '輸', 'description': 'Crispy脆樂團 - 輸', 'thumbnail': 're:^https?://.*\.jpg$', @@ -32,20 +31,19 @@ class StreetVoiceIE(InfoExtractor): song_id = self._match_id(url) song = self._download_json( - 'http://streetvoice.com/music/api/song/%s' % song_id, song_id) + 'https://streetvoice.com/api/v1/public/song/%s/' % song_id, song_id, data=b'') title = song['name'] - author = song['musician']['name'] + author = song['user']['nickname'] return { 'id': song_id, 'url': song['file'], - 'filesize': song.get('size'), 'title': title, 'description': '%s - %s' % (author, title), 'thumbnail': self._proto_relative_url(song.get('image'), 'http:'), 'duration': song.get('length'), 'upload_date': unified_strdate(song.get('created_at')), 'uploader': author, - 'uploader_id': compat_str(song['musician']['id']), + 'uploader_id': compat_str(song['user']['id']), } From 99ef96f84c2c9fa1267d6edb836c8f1945295424 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 21 Apr 2016 18:07:03 +0800 Subject: [PATCH 08/26] [gdcvault] Fix for videos with hard-coded hostnames Fixes #9248 --- youtube_dl/extractor/gdcvault.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py index 25e93c9a4..3ebcaf733 100644 --- a/youtube_dl/extractor/gdcvault.py +++ b/youtube_dl/extractor/gdcvault.py @@ -51,18 +51,34 @@ class GDCVaultIE(InfoExtractor): { 'url': 'http://gdcvault.com/play/1020791/', 'only_matching': True, - } + }, + { + 'url': 'http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface', + 'md5': 'a8efb6c31ed06ca8739294960b2dbabd', + 'info_dict': { + 'id': '1023460', + 'ext': 'mp4', + 'display_id': 'Tenacious-Design-and-The-Interface', + 'title': 'Tenacious Design and The Interface of \'Destiny\'', + }, + }, ] def _parse_mp4(self, xml_description): video_formats = [] - mp4_video = xml_description.find('./metadata/mp4video') - if mp4_video is None: - return None + video_root = None + + mp4_video = xml_description.find('./metadata/mp4video') + if mp4_video is not None: + mobj = re.match(r'(?Phttps?://.*?/).*', mp4_video.text) + video_root = mobj.group('root') + if video_root is None: + # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js + video_root = 'http://s3-2u.digitallyspeaking.com/' - mobj = re.match(r'(?Phttps?://.*?/).*', mp4_video.text) - video_root = mobj.group('root') formats = xml_description.findall('./metadata/MBRVideos/MBRVideo') + if not formats: + return None for format in formats: mobj = re.match(r'mp4\:(?P.*)', format.find('streamName').text) url = video_root + mobj.group('path') From ec59d657e7d898cce8f3a1b6556a79fd9495fc9d Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 21 Apr 2016 19:36:33 +0800 Subject: [PATCH 09/26] [dispeak] Add new extractor Both GDCVault and GPUTechConf uses the service of DigitalSpeaking. --- youtube_dl/extractor/dispeak.py | 111 ++++++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/gdcvault.py | 74 +------------------ youtube_dl/extractor/gputechconf.py | 36 ++------- 4 files changed, 123 insertions(+), 99 deletions(-) create mode 100644 youtube_dl/extractor/dispeak.py diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py new file mode 100644 index 000000000..6ebc3255a --- /dev/null +++ b/youtube_dl/extractor/dispeak.py @@ -0,0 +1,111 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + parse_duration, + remove_end, + xpath_element, + xpath_text, +) + + +class DigitalSpeakingIE(InfoExtractor): + _VALID_URL = r'http://evt.dispeak.com/([^/]+/)+xml/(?P[^.]+).xml' + + _TEST = { + # From http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml + 'url': 'http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml', + 'md5': 'a8efb6c31ed06ca8739294960b2dbabd', + 'info_dict': { + 'id': '840376_BQRC', + 'ext': 'mp4', + 'title': 'Tenacious Design and The Interface of \'Destiny\'', + }, + } + + def _parse_mp4(self, metadata): + video_formats = [] + video_root = None + + mp4_video = xpath_text(metadata, './mp4video', default=None) + if mp4_video is not None: + mobj = re.match(r'(?Phttps?://.*?/).*', mp4_video) + video_root = mobj.group('root') + if video_root is None: + http_host = xpath_text(metadata, 'httpHost', default=None) + if http_host: + video_root = 'http://%s/' % http_host + if video_root is None: + # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js + # Works for GPUTechConf, too + video_root = 'http://s3-2u.digitallyspeaking.com/' + + formats = metadata.findall('./MBRVideos/MBRVideo') + if not formats: + return None + for a_format in formats: + stream_name = xpath_text(a_format, 'streamName', fatal=True) + video_path = re.match(r'mp4\:(?P.*)', stream_name).group('path') + url = video_root + video_path + vbr = xpath_text(a_format, 'bitrate') + video_formats.append({ + 'url': url, + 'vbr': int_or_none(vbr), + }) + return video_formats + + def _parse_flv(self, metadata): + formats = [] + akamai_url = xpath_text(metadata, './akamaiHost', fatal=True) + audios = metadata.find('./audios') + if audios is not None: + for audio in audios: + formats.append({ + 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url, + 'play_path': remove_end(audio.get('url'), '.flv'), + 'ext': 'flv', + 'vcodec': 'none', + 'format_id': audio.get('code'), + }) + slide_video_path = xpath_text(metadata, './slideVideo', fatal=True) + formats.append({ + 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url, + 'play_path': remove_end(slide_video_path, '.flv'), + 'ext': 'flv', + 'format_note': 'slide deck video', + 'quality': -2, + 'preference': -2, + 'format_id': 'slides', + }) + speaker_video_path = xpath_text(metadata, './speakerVideo', fatal=True) + formats.append({ + 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url, + 'play_path': remove_end(speaker_video_path, '.flv'), + 'ext': 'flv', + 'format_note': 'speaker video', + 'quality': -1, + 'preference': -1, + 'format_id': 'speaker', + }) + return formats + + def _real_extract(self, url): + video_id = self._match_id(url) + + xml_description = self._download_xml(url, video_id) + metadata = xpath_element(xml_description, 'metadata') + + video_formats = self._parse_mp4(metadata) + if video_formats is None: + video_formats = self._parse_flv(metadata) + + return { + 'id': video_id, + 'formats': video_formats, + 'title': xpath_text(metadata, 'title', fatal=True), + 'duration': parse_duration(xpath_text(metadata, 'endTime')), + 'creator': xpath_text(metadata, 'speaker'), + } diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 1f3172119..84bdf5e97 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -197,6 +197,7 @@ from .dump import DumpIE from .dumpert import DumpertIE from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE +from .dispeak import DigitalSpeakingIE from .dropbox import DropboxIE from .dw import ( DWIE, diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py index 3ebcaf733..01e1ceec8 100644 --- a/youtube_dl/extractor/gdcvault.py +++ b/youtube_dl/extractor/gdcvault.py @@ -4,7 +4,6 @@ import re from .common import InfoExtractor from ..utils import ( - remove_end, HEADRequest, sanitized_Request, urlencode_postdata, @@ -64,66 +63,6 @@ class GDCVaultIE(InfoExtractor): }, ] - def _parse_mp4(self, xml_description): - video_formats = [] - video_root = None - - mp4_video = xml_description.find('./metadata/mp4video') - if mp4_video is not None: - mobj = re.match(r'(?Phttps?://.*?/).*', mp4_video.text) - video_root = mobj.group('root') - if video_root is None: - # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js - video_root = 'http://s3-2u.digitallyspeaking.com/' - - formats = xml_description.findall('./metadata/MBRVideos/MBRVideo') - if not formats: - return None - for format in formats: - mobj = re.match(r'mp4\:(?P.*)', format.find('streamName').text) - url = video_root + mobj.group('path') - vbr = format.find('bitrate').text - video_formats.append({ - 'url': url, - 'vbr': int(vbr), - }) - return video_formats - - def _parse_flv(self, xml_description): - formats = [] - akamai_url = xml_description.find('./metadata/akamaiHost').text - audios = xml_description.find('./metadata/audios') - if audios is not None: - for audio in audios: - formats.append({ - 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url, - 'play_path': remove_end(audio.get('url'), '.flv'), - 'ext': 'flv', - 'vcodec': 'none', - 'format_id': audio.get('code'), - }) - slide_video_path = xml_description.find('./metadata/slideVideo').text - formats.append({ - 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url, - 'play_path': remove_end(slide_video_path, '.flv'), - 'ext': 'flv', - 'format_note': 'slide deck video', - 'quality': -2, - 'preference': -2, - 'format_id': 'slides', - }) - speaker_video_path = xml_description.find('./metadata/speakerVideo').text - formats.append({ - 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url, - 'play_path': remove_end(speaker_video_path, '.flv'), - 'ext': 'flv', - 'format_note': 'speaker video', - 'quality': -1, - 'preference': -1, - 'format_id': 'speaker', - }) - return formats - def _login(self, webpage_url, display_id): (username, password) = self._get_login_info() if username is None or password is None: @@ -199,17 +138,10 @@ class GDCVaultIE(InfoExtractor): r'