From 36300346092937a8320fa4ae7303bc1b746a69ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 Mar 2015 03:30:18 +0600 Subject: [PATCH 001/118] [vk] Fix test (Closes #5100) --- youtube_dl/extractor/vk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py index 842263f34..cc384adbf 100644 --- a/youtube_dl/extractor/vk.py +++ b/youtube_dl/extractor/vk.py @@ -31,7 +31,7 @@ class VKIE(InfoExtractor): 'id': '162222515', 'ext': 'flv', 'title': 'ProtivoGunz - Хуёвая песня', - 'uploader': 're:Noize MC.*', + 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*', 'duration': 195, 'upload_date': '20120212', }, From 7594be85ffebee0c1ba457855ee7f4917e96499b Mon Sep 17 00:00:00 2001 From: Sergey Date: Mon, 2 Mar 2015 11:49:39 +0200 Subject: [PATCH 002/118] [lynda] Check for the empty subtitle --- youtube_dl/extractor/lynda.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py index 5dc22da22..a84019bdf 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dl/extractor/lynda.py @@ -144,6 +144,7 @@ class LyndaIE(InfoExtractor): def _fix_subtitles(self, subs): srt = '' + seq_counter = 0 for pos in range(0, len(subs) - 1): seq_current = subs[pos] m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode']) @@ -155,8 +156,10 @@ class LyndaIE(InfoExtractor): continue appear_time = m_current.group('timecode') disappear_time = m_next.group('timecode') - text = seq_current['Caption'].lstrip() - srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text) + text = seq_current['Caption'].strip() + if text: + seq_counter += 1 + srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (seq_counter, appear_time, disappear_time, text) if srt: return srt From 30cbd4e0d68361a20b036fc90bd53a8fa9a463dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 Mar 2015 22:12:10 +0600 Subject: [PATCH 003/118] [lynda] Completely skip videos we don't have access to, extract base class and modernize (Closes #5093) --- youtube_dl/extractor/lynda.py | 149 ++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 68 deletions(-) diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py index a84019bdf..1f02bef44 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dl/extractor/lynda.py @@ -15,18 +15,72 @@ from ..utils import ( ) -class LyndaIE(InfoExtractor): +class LyndaBaseIE(InfoExtractor): + _LOGIN_URL = 'https://www.lynda.com/login/login.aspx' + _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true' + _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.' + + def _real_initialize(self): + self._login() + + def _login(self): + (username, password) = self._get_login_info() + if username is None: + return + + login_form = { + 'username': username, + 'password': password, + 'remember': 'false', + 'stayPut': 'false' + } + request = compat_urllib_request.Request( + self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) + login_page = self._download_webpage( + request, None, 'Logging in as %s' % username) + + # Not (yet) logged in + m = re.search(r'loginResultJson = \'(?P[^\']+)\';', login_page) + if m is not None: + response = m.group('json') + response_json = json.loads(response) + state = response_json['state'] + + if state == 'notlogged': + raise ExtractorError( + 'Unable to login, incorrect username and/or password', + expected=True) + + # This is when we get popup: + # > You're already logged in to lynda.com on two devices. + # > If you log in here, we'll log you out of another device. + # So, we need to confirm this. + if state == 'conflicted': + confirm_form = { + 'username': '', + 'password': '', + 'resolve': 'true', + 'remember': 'false', + 'stayPut': 'false', + } + request = compat_urllib_request.Request( + self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form)) + login_page = self._download_webpage( + request, None, + 'Confirming log in and log out from another device') + + if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None: + raise ExtractorError('Unable to log in') + + +class LyndaIE(LyndaBaseIE): IE_NAME = 'lynda' IE_DESC = 'lynda.com videos' - _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(\d+)' - _LOGIN_URL = 'https://www.lynda.com/login/login.aspx' + _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P\d+)' _NETRC_MACHINE = 'lynda' - _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true' _TIMECODE_REGEX = r'\[(?P\d+:\d+:\d+[\.,]\d+)\]' - ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.' - _TESTS = [{ 'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html', 'md5': 'ecfc6862da89489161fb9cd5f5a6fac1', @@ -41,23 +95,22 @@ class LyndaIE(InfoExtractor): 'only_matching': True, }] - def _real_initialize(self): - self._login() - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1) + video_id = self._match_id(url) - page = self._download_webpage('http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id, video_id, - 'Downloading video JSON') + page = self._download_webpage( + 'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id, + video_id, 'Downloading video JSON') video_json = json.loads(page) if 'Status' in video_json: - raise ExtractorError('lynda returned error: %s' % video_json['Message'], expected=True) + raise ExtractorError( + 'lynda returned error: %s' % video_json['Message'], expected=True) if video_json['HasAccess'] is False: raise ExtractorError( - 'Video %s is only available for members. ' % video_id + self.ACCOUNT_CREDENTIALS_HINT, expected=True) + 'Video %s is only available for members. ' + % video_id + self._ACCOUNT_CREDENTIALS_HINT, expected=True) video_id = compat_str(video_json['ID']) duration = video_json['DurationInSeconds'] @@ -100,48 +153,6 @@ class LyndaIE(InfoExtractor): 'formats': formats } - def _login(self): - (username, password) = self._get_login_info() - if username is None: - return - - login_form = { - 'username': username, - 'password': password, - 'remember': 'false', - 'stayPut': 'false' - } - request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) - login_page = self._download_webpage(request, None, 'Logging in as %s' % username) - - # Not (yet) logged in - m = re.search(r'loginResultJson = \'(?P[^\']+)\';', login_page) - if m is not None: - response = m.group('json') - response_json = json.loads(response) - state = response_json['state'] - - if state == 'notlogged': - raise ExtractorError('Unable to login, incorrect username and/or password', expected=True) - - # This is when we get popup: - # > You're already logged in to lynda.com on two devices. - # > If you log in here, we'll log you out of another device. - # So, we need to confirm this. - if state == 'conflicted': - confirm_form = { - 'username': '', - 'password': '', - 'resolve': 'true', - 'remember': 'false', - 'stayPut': 'false', - } - request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form)) - login_page = self._download_webpage(request, None, 'Confirming log in and log out from another device') - - if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None: - raise ExtractorError('Unable to log in') - def _fix_subtitles(self, subs): srt = '' seq_counter = 0 @@ -172,7 +183,7 @@ class LyndaIE(InfoExtractor): return {} -class LyndaCourseIE(InfoExtractor): +class LyndaCourseIE(LyndaBaseIE): IE_NAME = 'lynda:course' IE_DESC = 'lynda.com online courses' @@ -185,35 +196,37 @@ class LyndaCourseIE(InfoExtractor): course_path = mobj.group('coursepath') course_id = mobj.group('courseid') - page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id, - course_id, 'Downloading course JSON') + page = self._download_webpage( + 'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id, + course_id, 'Downloading course JSON') course_json = json.loads(page) if 'Status' in course_json and course_json['Status'] == 'NotFound': - raise ExtractorError('Course %s does not exist' % course_id, expected=True) + raise ExtractorError( + 'Course %s does not exist' % course_id, expected=True) unaccessible_videos = 0 videos = [] - (username, _) = self._get_login_info() # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided # by single video API anymore for chapter in course_json['Chapters']: for video in chapter['Videos']: - if username is None and video['HasAccess'] is False: + if video['HasAccess'] is False: unaccessible_videos += 1 continue videos.append(video['ID']) if unaccessible_videos > 0: - self._downloader.report_warning('%s videos are only available for members and will not be downloaded. ' - % unaccessible_videos + LyndaIE.ACCOUNT_CREDENTIALS_HINT) + self._downloader.report_warning( + '%s videos are only available for members (or paid members) and will not be downloaded. ' + % unaccessible_videos + self._ACCOUNT_CREDENTIALS_HINT) entries = [ - self.url_result('http://www.lynda.com/%s/%s-4.html' % - (course_path, video_id), - 'Lynda') + self.url_result( + 'http://www.lynda.com/%s/%s-4.html' % (course_path, video_id), + 'Lynda') for video_id in videos] course_title = course_json['Title'] From 2f0f6578c3e4b82a5decc6f906a5f58b7b34c1a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 Mar 2015 22:38:44 +0600 Subject: [PATCH 004/118] [extractor/common] Assume non HTTP(S) URLs valid --- youtube_dl/extractor/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 7977fa8d0..cf39c0c21 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -767,6 +767,10 @@ class InfoExtractor(object): formats) def _is_valid_url(self, url, video_id, item='video'): + url = self._proto_relative_url(url, scheme='http:') + # For now assume non HTTP(S) URLs always valid + if not (url.startswith('http://') or url.startswith('https://')): + return True try: self._request_webpage(url, video_id, 'Checking %s URL' % item) return True From 562ceab13dc3a19243d5ab9d4d5927031a608334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 Mar 2015 22:39:32 +0600 Subject: [PATCH 005/118] [soundcloud] Check direct links validity (Closes #5101) --- youtube_dl/extractor/soundcloud.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index c5284fa67..95788098e 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -200,8 +200,9 @@ class SoundcloudIE(InfoExtractor): if f['format_id'].startswith('rtmp'): f['protocol'] = 'rtmp' - self._sort_formats(formats) - result['formats'] = formats + self._check_formats(formats, track_id) + self._sort_formats(formats) + result['formats'] = formats return result From 295df4edb97e3c0dc0ecd95746bb2c455607a4a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 Mar 2015 22:47:07 +0600 Subject: [PATCH 006/118] [soundcloud] Fix glitches (#5101) --- youtube_dl/extractor/soundcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 95788098e..9d4505972 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -180,7 +180,7 @@ class SoundcloudIE(InfoExtractor): 'format_id': key, 'url': url, 'play_path': 'mp3:' + path, - 'ext': ext, + 'ext': 'flv', 'vcodec': 'none', }) From c75f0b361a1b00f6ac1298615d6fee101994b2b9 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 2 Mar 2015 15:06:09 +0100 Subject: [PATCH 007/118] [downloader/external] Add support for custom options (Fixes #4885, closes #5098) --- youtube_dl/YoutubeDL.py | 4 ++-- youtube_dl/__init__.py | 5 +++++ youtube_dl/downloader/common.py | 2 ++ youtube_dl/downloader/external.py | 15 ++++++++++++--- youtube_dl/options.py | 6 +++++- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 15367c4e3..e0baa9872 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -249,10 +249,10 @@ class YoutubeDL(object): hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv. The following parameters are not used by YoutubeDL itself, they are used by - the FileDownloader: + the downloader (see youtube_dl/downloader/common.py): nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle, - xattr_set_filesize. + xattr_set_filesize, external_downloader_args. The following options are used by the post processors: prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available, diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 49f382695..6056da1be 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -9,6 +9,7 @@ import codecs import io import os import random +import shlex import sys @@ -255,6 +256,9 @@ def _real_main(argv=None): xattr # Confuse flake8 except ImportError: parser.error('setting filesize xattr requested but python-xattr is not available') + external_downloader_args = None + if opts.external_downloader_args: + external_downloader_args = shlex.split(opts.external_downloader_args) match_filter = ( None if opts.match_filter is None else match_filter_func(opts.match_filter)) @@ -359,6 +363,7 @@ def _real_main(argv=None): 'no_color': opts.no_color, 'ffmpeg_location': opts.ffmpeg_location, 'hls_prefer_native': opts.hls_prefer_native, + 'external_downloader_args': external_downloader_args, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 3ae90021a..8ed5c19a6 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -42,6 +42,8 @@ class FileDownloader(object): max_filesize: Skip files larger than this size xattr_set_filesize: Set ytdl.filesize user xattribute with expected size. (experimenatal) + external_downloader_args: A list of additional command-line arguments for the + external downloader. Subclasses of this one must re-define the real_download method. """ diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py index 51c41c704..1673b2382 100644 --- a/youtube_dl/downloader/external.py +++ b/youtube_dl/downloader/external.py @@ -51,6 +51,13 @@ class ExternalFD(FileDownloader): return [] return [command_option, source_address] + def _configuration_args(self, default=[]): + ex_args = self.params.get('external_downloader_args') + if ex_args is None: + return default + assert isinstance(ex_args, list) + return ex_args + def _call_downloader(self, tmpfilename, info_dict): """ Either overwrite this or implement _make_cmd """ cmd = self._make_cmd(tmpfilename, info_dict) @@ -79,6 +86,7 @@ class CurlFD(ExternalFD): for key, val in info_dict['http_headers'].items(): cmd += ['--header', '%s: %s' % (key, val)] cmd += self._source_address('--interface') + cmd += self._configuration_args() cmd += ['--', info_dict['url']] return cmd @@ -89,15 +97,16 @@ class WgetFD(ExternalFD): for key, val in info_dict['http_headers'].items(): cmd += ['--header', '%s: %s' % (key, val)] cmd += self._source_address('--bind-address') + cmd += self._configuration_args() cmd += ['--', info_dict['url']] return cmd class Aria2cFD(ExternalFD): def _make_cmd(self, tmpfilename, info_dict): - cmd = [ - self.exe, '-c', - '--min-split-size', '1M', '--max-connection-per-server', '4'] + cmd = [self.exe, '-c'] + cmd += self._configuration_args([ + '--min-split-size', '1M', '--max-connection-per-server', '4']) dn = os.path.dirname(tmpfilename) if dn: cmd += ['--dir', dn] diff --git a/youtube_dl/options.py b/youtube_dl/options.py index a2ffe96bc..df2be7b74 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -435,8 +435,12 @@ def parseOpts(overrideArguments=None): downloader.add_option( '--external-downloader', dest='external_downloader', metavar='COMMAND', - help='(experimental) Use the specified external downloader. ' + help='Use the specified external downloader. ' 'Currently supports %s' % ','.join(list_external_downloaders())) + downloader.add_option( + '--external-downloader-args', + dest='external_downloader_args', metavar='ARGS', + help='Give these arguments to the external downloader.') workarounds = optparse.OptionGroup(parser, 'Workarounds') workarounds.add_option( From 76c73715fb1e0eee61ace5ff7855d8237abdcd54 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 2 Mar 2015 15:21:11 +0100 Subject: [PATCH 008/118] [generic] Parse RSS enclosure URLs (Fixes #5091) --- youtube_dl/extractor/generic.py | 34 ++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 27e2bc300..5dc53685c 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -26,6 +26,7 @@ from ..utils import ( unsmuggle_url, UnsupportedError, url_basename, + xpath_text, ) from .brightcove import BrightcoveIE from .ooyala import OoyalaIE @@ -569,6 +570,16 @@ class GenericIE(InfoExtractor): 'title': 'John Carlson Postgame 2/25/15', }, }, + # RSS feed with enclosure + { + 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml', + 'info_dict': { + 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624', + 'ext': 'm4v', + 'upload_date': '20150228', + 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624', + } + } ] def report_following_redirect(self, new_url): @@ -580,11 +591,24 @@ class GenericIE(InfoExtractor): playlist_desc_el = doc.find('./channel/description') playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text - entries = [{ - '_type': 'url', - 'url': e.find('link').text, - 'title': e.find('title').text, - } for e in doc.findall('./channel/item')] + entries = [] + for it in doc.findall('./channel/item'): + next_url = xpath_text(it, 'link', fatal=False) + if not next_url: + enclosure_nodes = it.findall('./enclosure') + for e in enclosure_nodes: + next_url = e.attrib.get('url') + if next_url: + break + + if not next_url: + continue + + entries.append({ + '_type': 'url', + 'url': next_url, + 'title': it.find('title').text, + }) return { '_type': 'playlist', From a7440261c52d6518a429210aaae598095112c61b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 2 Mar 2015 19:07:17 +0100 Subject: [PATCH 009/118] [utils] Streap leading dots Fixes #2865, closes #5087 --- test/test_utils.py | 3 +++ youtube_dl/utils.py | 1 + 2 files changed, 4 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index 3fba8ae11..64fad58ad 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -85,8 +85,11 @@ class TestUtil(unittest.TestCase): self.assertEqual( sanitize_filename('New World record at 0:12:34'), 'New World record at 0_12_34') + self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf') self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf') + self.assertEqual(sanitize_filename('.gasdgf'), 'gasdgf') + self.assertEqual(sanitize_filename('.gasdgf', is_id=True), '.gasdgf') forbidden = '"\0\\/' for fc in forbidden: diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d4938ec36..1d3401bc2 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -305,6 +305,7 @@ def sanitize_filename(s, restricted=False, is_id=False): result = result[2:] if result.startswith('-'): result = '_' + result[len('-'):] + result = result.lstrip('.') if not result: result = '_' return result From 91410c9bfa9fd8f01fb817474bcc7b0db5cabf95 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 3 Mar 2015 00:03:06 +0100 Subject: [PATCH 010/118] [letv] Add --cn-verification-proxy (Closes #5077) --- youtube_dl/YoutubeDL.py | 5 ++++- youtube_dl/__init__.py | 1 + youtube_dl/extractor/letv.py | 34 ++++++++++++++++++++++++++++------ youtube_dl/options.py | 6 ++++++ youtube_dl/utils.py | 10 ++++++++++ 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e0baa9872..915963d96 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -54,6 +54,7 @@ from .utils import ( MaxDownloadsReached, PagedList, parse_filesize, + PerRequestProxyHandler, PostProcessingError, platform_name, preferredencoding, @@ -183,6 +184,8 @@ class YoutubeDL(object): prefer_insecure: Use HTTP instead of HTTPS to retrieve information. At the moment, this is only supported by YouTube. proxy: URL of the proxy server to use + cn_verification_proxy: URL of the proxy to use for IP address verification + on Chinese sites. (Experimental) socket_timeout: Time to wait for unresponsive hosts, in seconds bidi_workaround: Work around buggy terminals without bidirectional text support, using fridibi @@ -1762,7 +1765,7 @@ class YoutubeDL(object): # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805) if 'http' in proxies and 'https' not in proxies: proxies['https'] = proxies['http'] - proxy_handler = compat_urllib_request.ProxyHandler(proxies) + proxy_handler = PerRequestProxyHandler(proxies) debuglevel = 1 if self.params.get('debug_printtraffic') else 0 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 6056da1be..a08ddd670 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -364,6 +364,7 @@ def _real_main(argv=None): 'ffmpeg_location': opts.ffmpeg_location, 'hls_prefer_native': opts.hls_prefer_native, 'external_downloader_args': external_downloader_args, + 'cn_verification_proxy': opts.cn_verification_proxy, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py index 583ce35b9..fd5fd260e 100644 --- a/youtube_dl/extractor/letv.py +++ b/youtube_dl/extractor/letv.py @@ -7,8 +7,9 @@ import time from .common import InfoExtractor from ..compat import ( - compat_urlparse, compat_urllib_parse, + compat_urllib_request, + compat_urlparse, ) from ..utils import ( determine_ext, @@ -42,9 +43,23 @@ class LetvIE(InfoExtractor): 'expected_warnings': [ 'publish time' ] + }, { + 'note': 'This video is available only in Mainland China, thus a proxy is needed', + 'url': 'http://www.letv.com/ptv/vplay/1118082.html', + 'md5': 'f80936fbe20fb2f58648e81386ff7927', + 'info_dict': { + 'id': '1118082', + 'ext': 'mp4', + 'title': '与龙共舞 完整版', + 'description': 'md5:7506a5eeb1722bb9d4068f85024e3986', + }, + 'expected_warnings': [ + 'publish time' + ], + 'params': { + 'cn_verification_proxy': 'proxy.uku.im:8888' + }, }] - # http://www.letv.com/ptv/vplay/1118082.html - # This video is available only in Mainland China @staticmethod def urshift(val, n): @@ -76,8 +91,14 @@ class LetvIE(InfoExtractor): 'tkey': self.calc_time_key(int(time.time())), 'domain': 'www.letv.com' } + play_json_req = compat_urllib_request.Request( + 'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params) + ) + play_json_req.add_header( + 'Ytdl-Request-Proxy', + self._downloader.params.get('cn_verification_proxy')) play_json = self._download_json( - 'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params), + play_json_req, media_id, 'playJson data') # Check for errors @@ -114,7 +135,8 @@ class LetvIE(InfoExtractor): url_info_dict = { 'url': media_url, - 'ext': determine_ext(dispatch[format_id][1]) + 'ext': determine_ext(dispatch[format_id][1]), + 'format_id': format_id, } if format_id[-1:] == 'p': @@ -123,7 +145,7 @@ class LetvIE(InfoExtractor): urls.append(url_info_dict) publish_time = parse_iso8601(self._html_search_regex( - r'发布时间 ([^<>]+) ', page, 'publish time', fatal=False), + r'发布时间 ([^<>]+) ', page, 'publish time', default=None), delimiter=' ', timezone=datetime.timedelta(hours=8)) description = self._html_search_meta('description', page, fatal=False) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index df2be7b74..a4ca8adc4 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -195,6 +195,12 @@ def parseOpts(overrideArguments=None): action='store_const', const='::', dest='source_address', help='Make all connections via IPv6 (experimental)', ) + network.add_option( + '--cn-verification-proxy', + dest='cn_verification_proxy', default=None, metavar='URL', + help='Use this proxy to verify the IP address for some Chinese sites. ' + 'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)' + ) selection = optparse.OptionGroup(parser, 'Video Selection') selection.add_option( diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 1d3401bc2..b568288fa 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1768,3 +1768,13 @@ def match_filter_func(filter_str): video_title = info_dict.get('title', info_dict.get('id', 'video')) return '%s does not pass filter %s, skipping ..' % (video_title, filter_str) return _match_func + + +class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): + def proxy_open(self, req, proxy, type): + req_proxy = req.headers.get('Ytdl-Request-Proxy') + if req_proxy is not None: + proxy = req_proxy + del req.headers['Ytdl-Request-Proxy'] + return compat_urllib_request.ProxyHandler.proxy_open( + self, req, proxy, type) From 07490f8017a83b7607686499074c41212fa0a44f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 3 Mar 2015 00:05:05 +0100 Subject: [PATCH 011/118] release 2015.03.03 --- README.md | 411 ++++++++++++------------------------------ youtube_dl/version.py | 2 +- 2 files changed, 119 insertions(+), 294 deletions(-) diff --git a/README.md b/README.md index f2909e8d6..d05549125 100644 --- a/README.md +++ b/README.md @@ -47,211 +47,107 @@ which means you can modify it, redistribute it or use it however you like. # OPTIONS -h, --help print this help text and exit --version print program version and exit - -U, --update update this program to latest version. Make - sure that you have sufficient permissions - (run with sudo if needed) - -i, --ignore-errors continue on download errors, for example to - skip unavailable videos in a playlist - --abort-on-error Abort downloading of further videos (in the - playlist or the command line) if an error - occurs + -U, --update update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed) + -i, --ignore-errors continue on download errors, for example to skip unavailable videos in a playlist + --abort-on-error Abort downloading of further videos (in the playlist or the command line) if an error occurs --dump-user-agent display the current browser identification - --list-extractors List all supported extractors and the URLs - they would handle - --extractor-descriptions Output descriptions of all supported - extractors - --default-search PREFIX Use this prefix for unqualified URLs. For - example "gvsearch2:" downloads two videos - from google videos for youtube-dl "large - apple". Use the value "auto" to let - youtube-dl guess ("auto_warning" to emit a - warning when guessing). "error" just throws - an error. The default value "fixup_error" - repairs broken URLs, but emits an error if - this is not possible instead of searching. - --ignore-config Do not read configuration files. When given - in the global configuration file /etc - /youtube-dl.conf: Do not read the user - configuration in ~/.config/youtube- - dl/config (%APPDATA%/youtube-dl/config.txt - on Windows) - --flat-playlist Do not extract the videos of a playlist, - only list them. + --list-extractors List all supported extractors and the URLs they would handle + --extractor-descriptions Output descriptions of all supported extractors + --default-search PREFIX Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". + Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The + default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching. + --ignore-config Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration + in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows) + --flat-playlist Do not extract the videos of a playlist, only list them. --no-color Do not emit color codes in output. ## Network Options: - --proxy URL Use the specified HTTP/HTTPS proxy. Pass in - an empty string (--proxy "") for direct - connection + --proxy URL Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection --socket-timeout SECONDS Time to wait before giving up, in seconds - --source-address IP Client-side IP address to bind to - (experimental) - -4, --force-ipv4 Make all connections via IPv4 - (experimental) - -6, --force-ipv6 Make all connections via IPv6 - (experimental) + --source-address IP Client-side IP address to bind to (experimental) + -4, --force-ipv4 Make all connections via IPv4 (experimental) + -6, --force-ipv6 Make all connections via IPv6 (experimental) ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) --playlist-end NUMBER playlist video to end at (default is last) - --playlist-items ITEM_SPEC playlist video items to download. Specify - indices of the videos in the playlist - seperated by commas like: "--playlist-items - 1,2,5,8" if you want to download videos - indexed 1, 2, 5, 8 in the playlist. You can - specify range: "--playlist-items - 1-3,7,10-13", it will download the videos - at index 1, 2, 3, 7, 10, 11, 12 and 13. - --match-title REGEX download only matching titles (regex or - caseless sub-string) - --reject-title REGEX skip download for matching titles (regex or - caseless sub-string) + --playlist-items ITEM_SPEC playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" + if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will + download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13. + --match-title REGEX download only matching titles (regex or caseless sub-string) + --reject-title REGEX skip download for matching titles (regex or caseless sub-string) --max-downloads NUMBER Abort after downloading NUMBER files - --min-filesize SIZE Do not download any videos smaller than - SIZE (e.g. 50k or 44.6m) - --max-filesize SIZE Do not download any videos larger than SIZE - (e.g. 50k or 44.6m) + --min-filesize SIZE Do not download any videos smaller than SIZE (e.g. 50k or 44.6m) + --max-filesize SIZE Do not download any videos larger than SIZE (e.g. 50k or 44.6m) --date DATE download only videos uploaded in this date - --datebefore DATE download only videos uploaded on or before - this date (i.e. inclusive) - --dateafter DATE download only videos uploaded on or after - this date (i.e. inclusive) - --min-views COUNT Do not download any videos with less than - COUNT views - --max-views COUNT Do not download any videos with more than - COUNT views - --match-filter FILTER (Experimental) Generic video filter. - Specify any key (see help for -o for a list - of available keys) to match if the key is - present, !key to check if the key is not - present,key > NUMBER (like "comment_count > - 12", also works with >=, <, <=, !=, =) to - compare against a number, and & to require - multiple matches. Values which are not - known are excluded unless you put a - question mark (?) after the operator.For - example, to only match videos that have - been liked more than 100 times and disliked - less than 50 times (or the dislike - functionality is not available at the given - service), but who also have a description, - use --match-filter "like_count > 100 & + --datebefore DATE download only videos uploaded on or before this date (i.e. inclusive) + --dateafter DATE download only videos uploaded on or after this date (i.e. inclusive) + --min-views COUNT Do not download any videos with less than COUNT views + --max-views COUNT Do not download any videos with more than COUNT views + --match-filter FILTER (Experimental) Generic video filter. Specify any key (see help for -o for a list of available keys) to match if the key is present, + !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against + a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the + operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike + functionality is not available at the given service), but who also have a description, use --match-filter "like_count > 100 & dislike_count 10M]"). This works for - filesize, height, width, tbr, abr, vbr, - asr, and fps and the comparisons <, <=, >, - >=, =, != and for ext, acodec, vcodec, - container, and protocol and the comparisons - =, != . Formats for which the value is not - known are excluded unless you put a - question mark (?) after the operator. You - can combine format filters, so -f "[height - <=? 720][tbr>500]" selects up to 720p - videos (or videos where the height is not - known) with a bitrate of at least 500 - KBit/s. By default, youtube-dl will pick - the best quality. Use commas to download - multiple audio formats, such as -f - 136/137/mp4/bestvideo,140/m4a/bestaudio. - You can merge the video and audio of two - formats into a single file using -f + (requires ffmpeg or - avconv), for example -f + -f, --format FORMAT video format code, specify the order of preference using slashes, as in -f 22/17/18 . Instead of format codes, you can select by + extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio", + "worst". You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]"). + This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, + vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a + question mark (?) after the operator. You can combine format filters, so -f "[height <=? 720][tbr>500]" selects up to 720p videos + (or videos where the height is not known) with a bitrate of at least 500 KBit/s. By default, youtube-dl will pick the best quality. + Use commas to download multiple audio formats, such as -f 136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio + of two formats into a single file using -f + (requires ffmpeg or avconv), for example -f bestvideo+bestaudio. --all-formats download all available video formats - --prefer-free-formats prefer free video formats unless a specific - one is requested + --prefer-free-formats prefer free video formats unless a specific one is requested --max-quality FORMAT highest quality format to download -F, --list-formats list all available formats - --youtube-skip-dash-manifest Do not download the DASH manifest on - YouTube videos - --merge-output-format FORMAT If a merge is required (e.g. - bestvideo+bestaudio), output to given - container format. One of mkv, mp4, ogg, - webm, flv.Ignored if no merge is required + --youtube-skip-dash-manifest Do not download the DASH manifest on YouTube videos + --merge-output-format FORMAT If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no + merge is required ## Subtitle Options: --write-sub write subtitle file - --write-auto-sub write automatic subtitle file (youtube - only) - --all-subs downloads all the available subtitles of - the video + --write-auto-sub write automatic subtitle file (youtube only) + --all-subs downloads all the available subtitles of the video --list-subs lists all available subtitles for the video - --sub-format FORMAT subtitle format, accepts formats - preference, for example: "ass/srt/best" - --sub-lang LANGS languages of the subtitles to download - (optional) separated by commas, use IETF - language tags like 'en,pt' + --sub-format FORMAT subtitle format, accepts formats preference, for example: "ass/srt/best" + --sub-lang LANGS languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt' ## Authentication Options: -u, --username USERNAME login with this account ID - -p, --password PASSWORD account password. If this option is left - out, youtube-dl will ask interactively. + -p, --password PASSWORD account password. If this option is left out, youtube-dl will ask interactively. -2, --twofactor TWOFACTOR two-factor auth code -n, --netrc use .netrc authentication data --video-password PASSWORD video password (vimeo, smotri) ## Post-processing Options: - -x, --extract-audio convert video files to audio-only files - (requires ffmpeg or avconv and ffprobe or - avprobe) - --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", - "opus", or "wav"; "best" by default - --audio-quality QUALITY ffmpeg/avconv audio quality specification, - insert a value between 0 (better) and 9 - (worse) for VBR or a specific bitrate like - 128K (default 5) - --recode-video FORMAT Encode the video to another format if - necessary (currently supported: - mp4|flv|ogg|webm|mkv) - -k, --keep-video keeps the video file on disk after the - post-processing; the video is erased by - default - --no-post-overwrites do not overwrite post-processed files; the - post-processed files are overwritten by - default - --embed-subs embed subtitles in the video (only for mp4 - videos) + -x, --extract-audio convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe) + --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default + --audio-quality QUALITY ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K + (default 5) + --recode-video FORMAT Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv) + -k, --keep-video keeps the video file on disk after the post-processing; the video is erased by default + --no-post-overwrites do not overwrite post-processed files; the post-processed files are overwritten by default + --embed-subs embed subtitles in the video (only for mp4 videos) --embed-thumbnail embed thumbnail in the audio as cover art --add-metadata write metadata to the video file - --xattrs write metadata to the video file's xattrs - (using dublin core and xdg standards) - --fixup POLICY Automatically correct known faults of the - file. One of never (do nothing), warn (only - emit a warning), detect_or_warn(the - default; fix file if we can, warn - otherwise) - --prefer-avconv Prefer avconv over ffmpeg for running the - postprocessors (default) - --prefer-ffmpeg Prefer ffmpeg over avconv for running the - postprocessors - --ffmpeg-location PATH Location of the ffmpeg/avconv binary; - either the path to the binary or its - containing directory. - --exec CMD Execute a command on the file after - downloading, similar to find's -exec - syntax. Example: --exec 'adb push {} - /sdcard/Music/ && rm {}' - --convert-subtitles FORMAT Convert the subtitles to other format - (currently supported: srt|ass|vtt) + --xattrs write metadata to the video file's xattrs (using dublin core and xdg standards) + --fixup POLICY Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default; + fix file if we can, warn otherwise) + --prefer-avconv Prefer avconv over ffmpeg for running the postprocessors (default) + --prefer-ffmpeg Prefer ffmpeg over avconv for running the postprocessors + --ffmpeg-location PATH Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory. + --exec CMD Execute a command on the file after downloading, similar to find's -exec syntax. Example: --exec 'adb push {} /sdcard/Music/ && rm + {}' + --convert-subtitles FORMAT Convert the subtitles to other format (currently supported: srt|ass|vtt) # CONFIGURATION diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 5582348ba..3a49e52c9 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2015.02.28' +__version__ = '2015.03.03' From 499bfcbfd09e85f053d7e8943a8d47fed9349b0e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 3 Mar 2015 12:59:17 +0100 Subject: [PATCH 012/118] Make sure netrc works for all extractors with login support Fixes #5112 --- test/test_netrc.py | 26 ++++++++++++++++++++++++++ youtube_dl/extractor/atresplayer.py | 1 + youtube_dl/extractor/crunchyroll.py | 1 + youtube_dl/extractor/gdcvault.py | 1 + youtube_dl/extractor/lynda.py | 1 + youtube_dl/extractor/twitch.py | 1 + 6 files changed, 31 insertions(+) create mode 100644 test/test_netrc.py diff --git a/test/test_netrc.py b/test/test_netrc.py new file mode 100644 index 000000000..7cf3a6a2e --- /dev/null +++ b/test/test_netrc.py @@ -0,0 +1,26 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +from youtube_dl.extractor import ( + gen_extractors, +) + + +class TestNetRc(unittest.TestCase): + def test_netrc_present(self): + for ie in gen_extractors(): + if not hasattr(ie, '_login'): + continue + self.assertTrue( + hasattr(ie, '_NETRC_MACHINE'), + 'Extractor %s supports login, but is missing a _NETRC_MACHINE property' % ie.IE_NAME) + + +if __name__ == '__main__': + unittest.main() diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py index 7669e0e3d..29f8795d3 100644 --- a/youtube_dl/extractor/atresplayer.py +++ b/youtube_dl/extractor/atresplayer.py @@ -19,6 +19,7 @@ from ..utils import ( class AtresPlayerIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P.+?)_\d+\.html' + _NETRC_MACHINE = 'atresplayer' _TESTS = [ { 'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html', diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index f1da7d09b..e64b88fbc 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -29,6 +29,7 @@ from ..aes import ( class CrunchyrollIE(InfoExtractor): _VALID_URL = r'https?://(?:(?Pwww|m)\.)?(?Pcrunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P[0-9]+))(?:[/?&]|$)' + _NETRC_MACHINE = 'crunchyroll' _TESTS = [{ 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', 'info_dict': { diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py index f7b467b0a..51796f3a4 100644 --- a/youtube_dl/extractor/gdcvault.py +++ b/youtube_dl/extractor/gdcvault.py @@ -12,6 +12,7 @@ from ..utils import remove_end class GDCVaultIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P\d+)/(?P(\w|-)+)' + _NETRC_MACHINE = 'gdcvault' _TESTS = [ { 'url': 'http://www.gdcvault.com/play/1019721/Doki-Doki-Universe-Sweet-Simple', diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py index 1f02bef44..cfd3b14f4 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dl/extractor/lynda.py @@ -19,6 +19,7 @@ class LyndaBaseIE(InfoExtractor): _LOGIN_URL = 'https://www.lynda.com/login/login.aspx' _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true' _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.' + _NETRC_MACHINE = 'lynda' def _real_initialize(self): self._login() diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index 4b0ce54df..8af136147 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor): _API_BASE = 'https://api.twitch.tv' _USHER_BASE = 'http://usher.twitch.tv' _LOGIN_URL = 'https://secure.twitch.tv/user/login' + _NETRC_MACHINE = 'twitch' def _handle_error(self, response): if not isinstance(response, dict): From 2461f79d2ad9eee44644f6187e366125a29aa70f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 3 Mar 2015 13:56:06 +0100 Subject: [PATCH 013/118] [utils] Correct per-request proxy handling --- youtube_dl/YoutubeDL.py | 3 ++- youtube_dl/extractor/letv.py | 10 ++-------- youtube_dl/utils.py | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 915963d96..df2aebb59 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1771,7 +1771,8 @@ class YoutubeDL(object): https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel) ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel) opener = compat_urllib_request.build_opener( - https_handler, proxy_handler, cookie_processor, ydlh) + proxy_handler, https_handler, cookie_processor, ydlh) + # Delete the default user-agent header, which would otherwise apply in # cases where our custom HTTP handler doesn't come into play # (See https://github.com/rg3/youtube-dl/issues/1309 for details) diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py index fd5fd260e..85eee141b 100644 --- a/youtube_dl/extractor/letv.py +++ b/youtube_dl/extractor/letv.py @@ -40,9 +40,6 @@ class LetvIE(InfoExtractor): 'title': '美人天下01', 'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda', }, - 'expected_warnings': [ - 'publish time' - ] }, { 'note': 'This video is available only in Mainland China, thus a proxy is needed', 'url': 'http://www.letv.com/ptv/vplay/1118082.html', @@ -53,11 +50,8 @@ class LetvIE(InfoExtractor): 'title': '与龙共舞 完整版', 'description': 'md5:7506a5eeb1722bb9d4068f85024e3986', }, - 'expected_warnings': [ - 'publish time' - ], 'params': { - 'cn_verification_proxy': 'proxy.uku.im:8888' + 'cn_verification_proxy': 'http://proxy.uku.im:8888' }, }] @@ -95,7 +89,7 @@ class LetvIE(InfoExtractor): 'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params) ) play_json_req.add_header( - 'Ytdl-Request-Proxy', + 'Ytdl-request-proxy', self._downloader.params.get('cn_verification_proxy')) play_json = self._download_json( play_json_req, diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index b568288fa..7426e2a1f 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1771,10 +1771,21 @@ def match_filter_func(filter_str): class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): + def __init__(self, proxies=None): + # Set default handlers + for type in ('http', 'https'): + setattr(self, '%s_open' % type, + lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open: + meth(r, proxy, type)) + return compat_urllib_request.ProxyHandler.__init__(self, proxies) + def proxy_open(self, req, proxy, type): - req_proxy = req.headers.get('Ytdl-Request-Proxy') + req_proxy = req.headers.get('Ytdl-request-proxy') if req_proxy is not None: proxy = req_proxy - del req.headers['Ytdl-Request-Proxy'] + del req.headers['Ytdl-request-proxy'] + + if proxy == '__noproxy__': + return None # No Proxy return compat_urllib_request.ProxyHandler.proxy_open( self, req, proxy, type) From 7fde87c77da41f24faf34b37f5d05c029ac23d4f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 3 Mar 2015 13:59:38 +0100 Subject: [PATCH 014/118] release 2015.03.03.1 --- README.md | 2 ++ youtube_dl/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d05549125..5b9dd2cea 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ which means you can modify it, redistribute it or use it however you like. --source-address IP Client-side IP address to bind to (experimental) -4, --force-ipv4 Make all connections via IPv4 (experimental) -6, --force-ipv6 Make all connections via IPv6 (experimental) + --cn-verification-proxy URL Use this proxy to verify the IP address for some Chinese sites. The default proxy specified by --proxy (or none, if the options is + not present) is used for the actual downloading. (experimental) ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 3a49e52c9..252933993 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2015.03.03' +__version__ = '2015.03.03.1' From e0d0572b731b79393a5c5121acf4a43017160936 Mon Sep 17 00:00:00 2001 From: chaos33 <919946h43k48> Date: Tue, 3 Mar 2015 22:53:05 +0800 Subject: [PATCH 015/118] fix youporn extractor's json search regex --- youtube_dl/extractor/youporn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py index 107c9ac36..a278386d4 100644 --- a/youtube_dl/extractor/youporn.py +++ b/youtube_dl/extractor/youporn.py @@ -47,7 +47,7 @@ class YouPornIE(InfoExtractor): # Get JSON parameters json_params = self._search_regex( - r'var currentVideo = new Video\((.*)\)[,;]', + r'var videoJason = (.*)[,;]', webpage, 'JSON parameters') try: params = json.loads(json_params) From 50c9949d7ae8f76d7a105481e0e692de3448c385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 3 Mar 2015 21:39:04 +0600 Subject: [PATCH 016/118] [youporn] Imrove JSON regex and preserve the old one --- youtube_dl/extractor/youporn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py index a278386d4..e4c855ee0 100644 --- a/youtube_dl/extractor/youporn.py +++ b/youtube_dl/extractor/youporn.py @@ -47,7 +47,8 @@ class YouPornIE(InfoExtractor): # Get JSON parameters json_params = self._search_regex( - r'var videoJason = (.*)[,;]', + [r'var\s+videoJa?son\s*=\s*({.+?});', + r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'], webpage, 'JSON parameters') try: params = json.loads(json_params) From 22d362831978ea2b83ea0c21a634237dc2406ab6 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Tue, 3 Mar 2015 18:39:23 +0200 Subject: [PATCH 017/118] [tvplay] Adapt _VALID_URL and test case to domain name change --- youtube_dl/extractor/tvplay.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py index 9a53a3c74..e83e31a31 100644 --- a/youtube_dl/extractor/tvplay.py +++ b/youtube_dl/extractor/tvplay.py @@ -16,6 +16,7 @@ class TVPlayIE(InfoExtractor): _VALID_URL = r'''(?x)http://(?:www\.)? (?:tvplay\.lv/parraides| tv3play\.lt/programos| + play\.tv3\.lt/programos| tv3play\.ee/sisu| tv3play\.se/program| tv6play\.se/program| @@ -45,7 +46,7 @@ class TVPlayIE(InfoExtractor): }, }, { - 'url': 'http://www.tv3play.lt/programos/moterys-meluoja-geriau/409229?autostart=true', + 'url': 'http://play.tv3.lt/programos/moterys-meluoja-geriau/409229?autostart=true', 'info_dict': { 'id': '409229', 'ext': 'flv', From dc570c4951d397ded7535393a9f0972c9cd0bc04 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Tue, 3 Mar 2015 18:41:34 +0200 Subject: [PATCH 018/118] [lrt] Pass --realtime to rtmpdump --- youtube_dl/extractor/lrt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py index 9c2fbdd96..e3236f7b5 100644 --- a/youtube_dl/extractor/lrt.py +++ b/youtube_dl/extractor/lrt.py @@ -52,6 +52,7 @@ class LRTIE(InfoExtractor): 'url': data['streamer'], 'play_path': 'mp4:%s' % data['file'], 'preference': -1, + 'rtmp_real_time': True, }) else: formats.extend( From 123397317c9906bf53a23b6b426db0b08c70da32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 3 Mar 2015 18:45:56 +0100 Subject: [PATCH 019/118] [downloader/http] Remove wrong '_hook_progress' call (fixes #5117) --- youtube_dl/downloader/http.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 2e3dac825..5e9982799 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -218,12 +218,6 @@ class HttpFD(FileDownloader): if tmpfilename != '-': stream.close() - self._hook_progress({ - 'downloaded_bytes': byte_counter, - 'total_bytes': data_len, - 'tmpfilename': tmpfilename, - 'status': 'error', - }) if data_len is not None and byte_counter != data_len: raise ContentTooShortError(byte_counter, int(data_len)) self.try_rename(tmpfilename, filename) From f28fe66970370b5df649aa457e52cf99f6a49905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 4 Mar 2015 12:14:38 +0100 Subject: [PATCH 020/118] [downloader/http] Add missing fields for _hook_progress call It would fail if you run 'youtube-dl --no-part URL' a second time when the file has already been downloaded. (Reported in Fedora: https://bugzilla.redhat.com/show_bug.cgi?id=1195779) --- youtube_dl/downloader/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 5e9982799..4047d7167 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -92,6 +92,8 @@ class HttpFD(FileDownloader): self._hook_progress({ 'filename': filename, 'status': 'finished', + 'downloaded_bytes': resume_len, + 'total_bytes': resume_len, }) return True else: From 12a129ec6dde840c23e8b1e5d7552de93fce7be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 5 Mar 2015 02:36:53 +0600 Subject: [PATCH 021/118] [playwire] Add extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/playwire.py | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 youtube_dl/extractor/playwire.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ffcc7d9ab..5ca534cdf 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -364,6 +364,7 @@ from .planetaplay import PlanetaPlayIE from .played import PlayedIE from .playfm import PlayFMIE from .playvid import PlayvidIE +from .playwire import PlaywireIE from .podomatic import PodomaticIE from .pornhd import PornHdIE from .pornhub import ( diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py new file mode 100644 index 000000000..bdc71017b --- /dev/null +++ b/youtube_dl/extractor/playwire.py @@ -0,0 +1,78 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + xpath_text, + float_or_none, + int_or_none, +) + + +class PlaywireIE(InfoExtractor): + _VALID_URL = r'https?://(?:config|cdn)\.playwire\.com(?:/v2)?/(?P\d+)/(?:videos/v2|embed|config)/(?P\d+)' + _TESTS = [{ + 'url': 'http://config.playwire.com/14907/videos/v2/3353705/player.json', + 'md5': 'e6398701e3595888125729eaa2329ed9', + 'info_dict': { + 'id': '3353705', + 'ext': 'mp4', + 'title': 'S04_RM_UCL_Rus', + 'thumbnail': 're:^http://.*\.png$', + 'duration': 145.94, + }, + }, { + 'url': 'http://cdn.playwire.com/11625/embed/85228.html', + 'only_matching': True, + }, { + 'url': 'http://config.playwire.com/12421/videos/v2/3389892/zeus.json', + 'only_matching': True, + }, { + 'url': 'http://cdn.playwire.com/v2/12342/config/1532636.json', + 'only_matching': True, + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + publisher_id, video_id = mobj.group('publisher_id'), mobj.group('id') + + player = self._download_json( + 'http://config.playwire.com/%s/videos/v2/%s/zeus.json' % (publisher_id, video_id), + video_id) + + title = player['settings']['title'] + duration = float_or_none(player.get('duration'), 1000) + + content = player['content'] + thumbnail = content.get('poster') + src = content['media']['f4m'] + + f4m = self._download_xml(src, video_id) + base_url = xpath_text(f4m, './{http://ns.adobe.com/f4m/1.0}baseURL', 'base url', fatal=True) + formats = [] + for media in f4m.findall('./{http://ns.adobe.com/f4m/1.0}media'): + media_url = media.get('url') + if not media_url: + continue + tbr = int_or_none(media.get('bitrate')) + width = int_or_none(media.get('width')) + height = int_or_none(media.get('height')) + f = { + 'url': '%s/%s' % (base_url, media.attrib['url']), + 'tbr': tbr, + 'width': width, + 'height': height, + } + if not (tbr or width or height): + f['quality'] = 1 if '-hd.' in media_url else 0 + formats.append(f) + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'thumbnail': thumbnail, + 'duration': duration, + 'formats': formats, + } From c2ebea6580481a89c35f9447898b17241ad71ac0 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Thu, 5 Mar 2015 14:45:38 +0200 Subject: [PATCH 022/118] [extremetube] Fix extraction (Closes #5127) --- youtube_dl/extractor/extremetube.py | 30 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py index 36ba33128..c826a5404 100644 --- a/youtube_dl/extractor/extremetube.py +++ b/youtube_dl/extractor/extremetube.py @@ -4,11 +4,11 @@ import re from .common import InfoExtractor from ..compat import ( - compat_urllib_parse_urlparse, + compat_parse_qs, compat_urllib_request, - compat_urllib_parse, ) from ..utils import ( + qualities, str_to_int, ) @@ -17,7 +17,7 @@ class ExtremeTubeIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?(?Pextremetube\.com/.*?video/.+?(?P[0-9]+))(?:[/?&]|$)' _TESTS = [{ 'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431', - 'md5': '1fb9228f5e3332ec8c057d6ac36f33e0', + 'md5': '344d0c6d50e2f16b06e49ca011d8ac69', 'info_dict': { 'id': '652431', 'ext': 'mp4', @@ -49,19 +49,27 @@ class ExtremeTubeIE(InfoExtractor): r'Views:\s*\s*([\d,\.]+)', webpage, 'view count', fatal=False)) - video_url = compat_urllib_parse.unquote(self._html_search_regex( - r'video_url=(.+?)&', webpage, 'video_url')) - path = compat_urllib_parse_urlparse(video_url).path - format = path.split('/')[5].split('_')[:2] - format = "-".join(format) + flash_vars = compat_parse_qs(self._search_regex( + r']+?name="flashvars"[^>]+?value="([^"]+)"', webpage, 'flash vars')) + + formats = [] + quality = qualities(['180p', '240p', '360p', '480p', '720p', '1080p']) + for k, vals in flash_vars.items(): + m = re.match(r'quality_(?P[0-9]+p)$', k) + if m is not None: + formats.append({ + 'format_id': m.group('quality'), + 'quality': quality(m.group('quality')), + 'url': vals[0], + }) + + self._sort_formats(formats) return { 'id': video_id, 'title': video_title, + 'formats': formats, 'uploader': uploader, 'view_count': view_count, - 'url': video_url, - 'format': format, - 'format_id': format, 'age_limit': 18, } From 9609f02e3ca87e56fc5571e41040ddbcf2a79a73 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 5 Mar 2015 22:34:56 +0100 Subject: [PATCH 023/118] [vidme] Modernize --- youtube_dl/extractor/vidme.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py index 5c89824c1..339c3d897 100644 --- a/youtube_dl/extractor/vidme.py +++ b/youtube_dl/extractor/vidme.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..utils import ( int_or_none, @@ -28,12 +26,11 @@ class VidmeIE(InfoExtractor): } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - video_url = self._html_search_regex(r' Date: Fri, 6 Mar 2015 10:44:24 +0100 Subject: [PATCH 024/118] [travis] Declare 3.2 (Fixes #5144) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fb34299fc..511bee64c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.2" - "3.3" - "3.4" before_install: From a0bb7c5593b08339f34dc43d9bc1b199ddc3ff7d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 6 Mar 2015 10:49:42 +0100 Subject: [PATCH 025/118] [extractor/common] Improve m3u format IDs (#5143) --- youtube_dl/extractor/common.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index cf39c0c21..8ff76342f 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -883,8 +883,12 @@ class InfoExtractor(object): formats.append({'url': format_url(line)}) continue tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000) + format_id = last_media.get('NAME') + if not format_id: + format_id = '-'.join(filter(None, [ + m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])) f = { - 'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])), + 'format_id': format_id, 'url': format_url(line.strip()), 'tbr': tbr, 'ext': ext, From d0e958c71c8c561d1c3917d114370fcb85972181 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 6 Mar 2015 10:53:49 +0100 Subject: [PATCH 026/118] [twitch:vod] Prefer source stream (Fixes #5143) --- youtube_dl/extractor/twitch.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index 8af136147..b058891bd 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -85,6 +85,14 @@ class TwitchBaseIE(InfoExtractor): raise ExtractorError( 'Unable to login: %s' % m.group('msg').strip(), expected=True) + def _prefer_source(self, formats): + try: + source = next(f for f in formats if f['format_id'] == 'Source') + source['preference'] = 10 + except StopIteration: + pass # No Source stream present + self._sort_formats(formats) + class TwitchItemBaseIE(TwitchBaseIE): def _download_info(self, item, item_id): @@ -209,6 +217,7 @@ class TwitchVodIE(TwitchItemBaseIE): '%s/vod/%s?nauth=%s&nauthsig=%s' % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']), item_id, 'mp4') + self._prefer_source(formats) info['formats'] = formats return info @@ -357,13 +366,7 @@ class TwitchStreamIE(TwitchBaseIE): '%s/api/channel/hls/%s.m3u8?%s' % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')), channel_id, 'mp4') - - # prefer the 'source' stream, the others are limited to 30 fps - def _sort_source(f): - if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source': - return 1 - return 0 - formats = sorted(formats, key=_sort_source) + self._prefer_source(formats) view_count = stream.get('viewers') timestamp = parse_iso8601(stream.get('created_at')) From 8dc9d361c22754e6b6b52136d496d754dea65b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 6 Mar 2015 22:52:50 +0600 Subject: [PATCH 027/118] [extractor/common] Fix format_id when `last_media` is None and always include `m3u8_id` if present The rationale behind `m3u8_id` was to resolve duplicates when processing several m3u8 playlists within the same media that give equal resulting `format_id`'s, e.g. `youtube-dl http://www.rts.ch/play/tv/passe-moi-les-jumelles/video/la-fee-des-bois-mustang-les-chemins-du-vent?id=3854925 -F` --- youtube_dl/extractor/common.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 8ff76342f..06bce12e0 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -883,12 +883,13 @@ class InfoExtractor(object): formats.append({'url': format_url(line)}) continue tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000) - format_id = last_media.get('NAME') - if not format_id: - format_id = '-'.join(filter(None, [ - m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])) + format_id = [] + if m3u8_id: + format_id.append(m3u8_id) + last_media_name = last_media.get('NAME') if last_media else None + format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats))) f = { - 'format_id': format_id, + 'format_id': '-'.join(format_id), 'url': format_url(line.strip()), 'tbr': tbr, 'ext': ext, From bd05aa4e24774a9aa9f68a8379900e6b756aae11 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 7 Mar 2015 00:53:52 +0800 Subject: [PATCH 028/118] [Yam] Add an error detection and update test cases --- youtube_dl/extractor/yam.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py index b294767c5..19ad74d04 100644 --- a/youtube_dl/extractor/yam.py +++ b/youtube_dl/extractor/yam.py @@ -8,6 +8,7 @@ from ..compat import compat_urlparse from ..utils import ( float_or_none, month_by_abbreviation, + ExtractorError, ) @@ -28,23 +29,44 @@ class YamIE(InfoExtractor): } }, { # An external video hosted on YouTube - 'url': 'http://mymedia.yam.com/m/3598173', - 'md5': '0238ceec479c654e8c2f1223755bf3e9', + 'url': 'http://mymedia.yam.com/m/3599430', + 'md5': '03127cf10d8f35d120a9e8e52e3b17c6', 'info_dict': { - 'id': 'pJ2Deys283c', + 'id': 'CNpEoQlrIgA', 'ext': 'mp4', - 'upload_date': '20150202', + 'upload_date': '20150306', 'uploader': '新莊社大瑜伽社', - 'description': 'md5:f5cc72f0baf259a70fb731654b0d2eff', + 'description': 'md5:11e2e405311633ace874f2e6226c8b17', 'uploader_id': '2323agoy', - 'title': '外婆的澎湖灣KTV-潘安邦', + 'title': '20090412陽明山二子坪-1', } + }, { + 'url': 'http://mymedia.yam.com/m/3598173', + 'info_dict': { + 'id': '3598173', + 'ext': 'mp4', + }, + 'skip': 'cause Yam system error', + }, { + 'url': 'http://mymedia.yam.com/m/3599437', + 'info_dict': { + 'id': '3599437', + 'ext': 'mp4', + }, + 'skip': 'invalid YouTube URL', }] def _real_extract(self, url): video_id = self._match_id(url) page = self._download_webpage(url, video_id) + # Check for errors + system_msg = self._html_search_regex( + r'系統訊息(?:
|\n|\r)*([^<>]+)
', page, 'system message', + default=None) + if system_msg: + raise ExtractorError(system_msg, expected=True) + # Is it hosted externally on YouTube? youtube_url = self._html_search_regex( r' Date: Fri, 6 Mar 2015 22:53:53 +0600 Subject: [PATCH 029/118] [extractor/common] Remove 'm3u8' from quality selection URL --- youtube_dl/extractor/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 06bce12e0..f9e8e2bad 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -839,7 +839,7 @@ class InfoExtractor(object): m3u8_id=None): formats = [{ - 'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])), + 'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])), 'url': m3u8_url, 'ext': ext, 'protocol': 'm3u8', From 3c6f245083db85d7ba9ae5444561339dc1b02671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Mar 2015 18:16:56 +0100 Subject: [PATCH 030/118] [vimeo] Fix upload date extraction --- youtube_dl/extractor/vimeo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 8f540f578..3771769e1 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -20,6 +20,7 @@ from ..utils import ( RegexNotFoundError, smuggle_url, std_headers, + unified_strdate, unsmuggle_url, urlencode_postdata, ) @@ -140,6 +141,7 @@ class VimeoIE(VimeoBaseInfoExtractor): 'description': 'md5:8678b246399b070816b12313e8b4eb5c', 'uploader_id': 'atencio', 'uploader': 'Peter Atencio', + 'upload_date': '20130927', 'duration': 187, }, }, @@ -323,9 +325,9 @@ class VimeoIE(VimeoBaseInfoExtractor): # Extract upload date video_upload_date = None - mobj = re.search(r' Date: Fri, 6 Mar 2015 19:08:27 +0100 Subject: [PATCH 031/118] [vimeo] Fix and use '_verify_video_password' (#5001) It only supports verifying the password over https now. Use it instead of manually setting the 'password' cookie because it allows to check if the password is correct. --- youtube_dl/extractor/vimeo.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 3771769e1..61f2d1ee8 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import json import re import itertools -import hashlib from .common import InfoExtractor from ..compat import ( @@ -178,17 +177,15 @@ class VimeoIE(VimeoBaseInfoExtractor): password = self._downloader.params.get('videopassword', None) if password is None: raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True) - token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token') - data = compat_urllib_parse.urlencode({ + token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token') + data = urlencode_postdata({ 'password': password, 'token': token, }) - # I didn't manage to use the password with https - if url.startswith('https'): - pass_url = url.replace('https', 'http') - else: - pass_url = url - password_request = compat_urllib_request.Request(pass_url + '/password', data) + if url.startswith('http://'): + # vimeo only supports https now, but the user can give an http url + url = url.replace('http://', 'https://') + password_request = compat_urllib_request.Request(url + '/password', data) password_request.add_header('Content-Type', 'application/x-www-form-urlencoded') password_request.add_header('Cookie', 'xsrft=%s' % token) return self._download_webpage( @@ -227,11 +224,6 @@ class VimeoIE(VimeoBaseInfoExtractor): if mobj.group('pro') or mobj.group('player'): url = 'http://player.vimeo.com/video/' + video_id - password = self._downloader.params.get('videopassword', None) - if password: - headers['Cookie'] = '%s_password=%s' % ( - video_id, hashlib.md5(password.encode('utf-8')).hexdigest()) - # Retrieve video webpage to extract further information request = compat_urllib_request.Request(url, None, headers) try: From d1508cd68d0a3e470860dc7f225509b2e422d34b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Mar 2015 22:16:26 +0100 Subject: [PATCH 032/118] [vimeo:album] Fix password protected videos Since it only uses https now, don't recognize http urls. --- youtube_dl/extractor/vimeo.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 61f2d1ee8..0d1c92633 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -429,10 +429,10 @@ class VimeoChannelIE(InfoExtractor): name="([^"]+)"\s+ value="([^"]*)" ''', login_form)) - token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token') + token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token') fields['token'] = token fields['password'] = password - post = compat_urllib_parse.urlencode(fields) + post = urlencode_postdata(fields) password_path = self._search_regex( r'action="([^"]+)"', login_form, 'password URL') password_url = compat_urlparse.urljoin(page_url, password_path) @@ -494,10 +494,10 @@ class VimeoUserIE(VimeoChannelIE): class VimeoAlbumIE(VimeoChannelIE): IE_NAME = 'vimeo:album' - _VALID_URL = r'https?://vimeo\.com/album/(?P\d+)' + _VALID_URL = r'https://vimeo\.com/album/(?P\d+)' _TITLE_RE = r'
\n\s*

(.*?)

' _TESTS = [{ - 'url': 'http://vimeo.com/album/2632481', + 'url': 'https://vimeo.com/album/2632481', 'info_dict': { 'id': '2632481', 'title': 'Staff Favorites: November 2013', @@ -521,7 +521,7 @@ class VimeoAlbumIE(VimeoChannelIE): def _real_extract(self, url): album_id = self._match_id(url) - return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id) + return self._extract_videos(album_id, 'https://vimeo.com/album/%s' % album_id) class VimeoGroupsIE(VimeoAlbumIE): From 61e00a97756de178db3e2a55b4292273532c9420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Mar 2015 22:39:05 +0100 Subject: [PATCH 033/118] [vimeo] Use https for player.vimeo.com urls (closes #5147) --- youtube_dl/extractor/vimeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 0d1c92633..b84a83ba6 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor): video_id = mobj.group('id') orig_url = url if mobj.group('pro') or mobj.group('player'): - url = 'http://player.vimeo.com/video/' + video_id + url = 'https://player.vimeo.com/video/' + video_id # Retrieve video webpage to extract further information request = compat_urllib_request.Request(url, None, headers) From 23ba76bc0eae4931c1adbd00de45a9ed41848283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Mar 2015 22:45:05 +0100 Subject: [PATCH 034/118] [dailymotion] Replace test It has been removed. --- youtube_dl/extractor/dailymotion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 42b20a46d..4f67c3aac 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -46,13 +46,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor): _TESTS = [ { - 'url': 'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech', - 'md5': '392c4b85a60a90dc4792da41ce3144eb', + 'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames', + 'md5': '2137c41a8e78554bb09225b8eb322406', 'info_dict': { - 'id': 'x33vw9', + 'id': 'x2iuewm', 'ext': 'mp4', - 'uploader': 'Amphora Alex and Van .', - 'title': 'Tutoriel de Youtubeur"DL DES VIDEO DE YOUTUBE"', + 'uploader': 'IGN', + 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News', } }, # Vevo video From 14137b5781305d3c9db523b14283128335f62135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 7 Mar 2015 17:31:03 +0600 Subject: [PATCH 035/118] [orf:iptv] Add extractor (Closes #5140) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/orf.py | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 5ca534cdf..f72d06fc4 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -354,6 +354,7 @@ from .orf import ( ORFTVthekIE, ORFOE1IE, ORFFM4IE, + ORFIPTVIE, ) from .parliamentliveuk import ParliamentLiveUKIE from .patreon import PatreonIE diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index 4e293392b..ca1a5bb3c 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -11,6 +11,11 @@ from ..utils import ( HEADRequest, unified_strdate, ExtractorError, + strip_jsonp, + int_or_none, + float_or_none, + determine_ext, + remove_end, ) @@ -197,3 +202,92 @@ class ORFFM4IE(InfoExtractor): 'description': data['subtitle'], 'entries': entries } + + +class ORFIPTVIE(InfoExtractor): + IE_NAME = 'orf:iptv' + IE_DESC = 'iptv.ORF.at' + _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P\d+)' + + _TEST = { + 'url': 'http://iptv.orf.at/stories/2267952', + 'md5': '26ffa4bab6dbce1eee78bbc7021016cd', + 'info_dict': { + 'id': '339775', + 'ext': 'flv', + 'title': 'Kreml-Kritiker Nawalny wieder frei', + 'description': 'md5:6f24e7f546d364dacd0e616a9e409236', + 'duration': 84.729, + 'thumbnail': 're:^https?://.*\.jpg$', + 'upload_date': '20150306', + }, + } + + def _real_extract(self, url): + story_id = self._match_id(url) + + webpage = self._download_webpage( + 'http://iptv.orf.at/stories/%s' % story_id, story_id) + + video_id = self._search_regex( + r'data-video(?:id)?="(\d+)"', webpage, 'video id') + + data = self._download_json( + 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id, + video_id)[0] + + duration = float_or_none(data['duration'], 1000) + + video = data['sources']['default'] + load_balancer_url = video['loadBalancerUrl'] + abr = int_or_none(video.get('audioBitrate')) + vbr = int_or_none(video.get('bitrate')) + fps = int_or_none(video.get('videoFps')) + width = int_or_none(video.get('videoWidth')) + height = int_or_none(video.get('videoHeight')) + thumbnail = video.get('preview') + + rendition = self._download_json( + load_balancer_url, video_id, transform_source=strip_jsonp) + + f = { + 'abr': abr, + 'vbr': vbr, + 'fps': fps, + 'width': width, + 'height': height, + } + + formats = [] + for format_id, format_url in rendition['redirect'].items(): + if format_id == 'rtmp': + ff = f.copy() + ff.update({ + 'url': format_url, + 'format_id': format_id, + }) + formats.append(ff) + elif determine_ext(format_url) == 'f4m': + formats.extend(self._extract_f4m_formats( + format_url, video_id, f4m_id=format_id)) + elif determine_ext(format_url) == 'm3u8': + formats.extend(self._extract_m3u8_formats( + format_url, video_id, 'mp4', m3u8_id=format_id)) + else: + continue + self._sort_formats(formats) + + title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at') + description = self._og_search_description(webpage) + upload_date = unified_strdate(self._html_search_meta( + 'dc.date', webpage, 'upload date')) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'duration': duration, + 'thumbnail': thumbnail, + 'upload_date': upload_date, + 'formats': formats, + } From 98998cded6e179b35055177f75fc921d99ee5938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 7 Mar 2015 18:59:06 +0600 Subject: [PATCH 036/118] [youtube:search_url] Fix extraction (Closes #5155) --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 3690f8021..27c8c4453 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1532,7 +1532,7 @@ class YoutubeSearchURLIE(InfoExtractor): webpage = self._download_webpage(url, query) result_code = self._search_regex( - r'(?s)
    ', webpage, 'result HTML') + r'(?s)]+class="item-section"(.*?)
', webpage, 'result HTML') part_codes = re.findall( r'(?s)

(.*?)

', result_code) From 0bf79ac455ff5dc52b61f26c254f83e7c041d75c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 7 Mar 2015 22:16:23 +0600 Subject: [PATCH 037/118] [eagleplatform] Add extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/eagleplatform.py | 99 +++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 youtube_dl/extractor/eagleplatform.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index f72d06fc4..b489d5770 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -116,6 +116,7 @@ from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE from .divxstage import DivxStageIE from .dropbox import DropboxIE +from .eagleplatform import EaglePlatformIE from .ebaumsworld import EbaumsWorldIE from .echomsk import EchoMskIE from .ehow import EHowIE diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py new file mode 100644 index 000000000..654f746fe --- /dev/null +++ b/youtube_dl/extractor/eagleplatform.py @@ -0,0 +1,99 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + int_or_none, +) + + +class EaglePlatformIE(InfoExtractor): + _VALID_URL = r'''(?x) + (?: + eagleplatform:(?P[^/]+):| + https?://(?P.+?\.media\.eagleplatform\.com)/index/player\?.*\brecord_id= + ) + (?P\d+) + ''' + _TESTS = [{ + # http://lenta.ru/news/2015/03/06/navalny/ + 'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201', + 'md5': '0b7994faa2bd5c0f69a3db6db28d078d', + 'info_dict': { + 'id': '227304', + 'ext': 'mp4', + 'title': 'Навальный вышел на свободу', + 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 87, + 'view_count': int, + 'age_limit': 0, + }, + }, { + # http://muz-tv.ru/play/7129/ + # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true + 'url': 'eagleplatform:media.clipyou.ru:12820', + 'md5': '6c2ebeab03b739597ce8d86339d5a905', + 'info_dict': { + 'id': '12820', + 'ext': 'mp4', + 'title': "'O Sole Mio", + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 216, + 'view_count': int, + }, + }] + + def _handle_error(self, response): + print(response['status']) + status = int_or_none(response.get('status', 200)) + if status != 200: + raise ExtractorError(' '.join(response['errors']), expected=True) + + def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'): + response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note) + self._handle_error(response) + return response + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id') + + player_data = self._download_json( + 'http://%s/api/player_data?id=%s' % (host, video_id), video_id) + + media = player_data['data']['playlist']['viewports'][0]['medialist'][0] + + title = media['title'] + description = media.get('description') + thumbnail = media.get('snapshot') + duration = int_or_none(media.get('duration')) + view_count = int_or_none(media.get('views')) + + age_restriction = media.get('age_restriction') + age_limit = None + if age_restriction: + age_limit = 0 if age_restriction == 'allow_all' else 18 + + m3u8_data = self._download_json( + media['sources']['secure_m3u8']['auto'], + video_id, 'Downloading m3u8 JSON') + + formats = self._extract_m3u8_formats( + m3u8_data['data'][0], video_id, + 'mp4', entry_protocol='m3u8_native') + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'view_count': view_count, + 'age_limit': age_limit, + 'formats': formats, + } From 135c9c42bf272c667f38a0a42e97e381fe53df91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 7 Mar 2015 22:22:57 +0600 Subject: [PATCH 038/118] [eagleplatform] Add support for embeds --- youtube_dl/extractor/generic.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 5dc53685c..a12b29da1 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -570,6 +570,20 @@ class GenericIE(InfoExtractor): 'title': 'John Carlson Postgame 2/25/15', }, }, + # Eagle.Platform embed (generic URL) + { + 'url': 'http://lenta.ru/news/2015/03/06/navalny/', + 'info_dict': { + 'id': '227304', + 'ext': 'mp4', + 'title': 'Навальный вышел на свободу', + 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 87, + 'view_count': int, + 'age_limit': 0, + }, + }, # RSS feed with enclosure { 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml', @@ -1155,6 +1169,12 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura') + # Look for Eagle.Platform embeds + mobj = re.search( + r']+src="(?Phttps?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'EaglePlatform') + def check_video(vurl): if YoutubeIE.suitable(vurl): return True From d47ae7f620e3b7be2387da2a61b618a3730d932a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 7 Mar 2015 22:34:44 +0600 Subject: [PATCH 039/118] [eagleplatform] Add support for ClipYou embeds --- youtube_dl/extractor/generic.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index a12b29da1..013198b0d 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -584,6 +584,18 @@ class GenericIE(InfoExtractor): 'age_limit': 0, }, }, + # ClipYou (Eagle.Platform) embed (custom URL) + { + 'url': 'http://muz-tv.ru/play/7129/', + 'info_dict': { + 'id': '12820', + 'ext': 'mp4', + 'title': "'O Sole Mio", + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 216, + 'view_count': int, + }, + }, # RSS feed with enclosure { 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml', @@ -1175,6 +1187,12 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group('url'), 'EaglePlatform') + # Look for ClipYou (uses Eagle.Platform) embeds + mobj = re.search( + r']+src="https?://(?Pmedia\.clipyou\.ru)/index/player\?.*\brecord_id=(?P\d+).*"', webpage) + if mobj is not None: + return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform') + def check_video(vurl): if YoutubeIE.suitable(vurl): return True From 8a48223a7ba5a3d1e47a36c30dd925e11183ae63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 7 Mar 2015 22:35:36 +0600 Subject: [PATCH 040/118] [eagleplatform] Remove debug output --- youtube_dl/extractor/eagleplatform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py index 654f746fe..7173371ee 100644 --- a/youtube_dl/extractor/eagleplatform.py +++ b/youtube_dl/extractor/eagleplatform.py @@ -48,7 +48,6 @@ class EaglePlatformIE(InfoExtractor): }] def _handle_error(self, response): - print(response['status']) status = int_or_none(response.get('status', 200)) if status != 200: raise ExtractorError(' '.join(response['errors']), expected=True) From 7dcad95d4faa91adfaa4d87d411c4bc55ab000e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 01:13:23 +0600 Subject: [PATCH 041/118] [YoutubeDL] Sanitize outtmpl as it may contain forbidden characters --- youtube_dl/YoutubeDL.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index df2aebb59..bae52e9c7 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -553,16 +553,20 @@ class YoutubeDL(object): elif template_dict.get('width'): template_dict['resolution'] = '?x%d' % template_dict['width'] + restrict_filenames = self.params.get('restrictfilenames') + sanitize = lambda k, v: sanitize_filename( compat_str(v), - restricted=self.params.get('restrictfilenames'), + restricted=restrict_filenames, is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() if v is not None) template_dict = collections.defaultdict(lambda: 'NA', template_dict) - outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) + outtmpl = sanitize_filename( + self.params.get('outtmpl', DEFAULT_OUTTMPL), + restricted=restrict_filenames) tmpl = compat_expanduser(outtmpl) filename = tmpl % template_dict # Temporary fix for #4787 From ab205b9dc8dcee37786dac597f2c3760c89b191c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 7 Mar 2015 22:18:16 +0100 Subject: [PATCH 042/118] Revert "[YoutubeDL] Sanitize outtmpl as it may contain forbidden characters" This reverts commit 7dcad95d4faa91adfaa4d87d411c4bc55ab000e4. The output template is most definitly allowed to contain forbidden characters; otherwise -o /foo/bar/vid.mp4 wouldn't work. --- youtube_dl/YoutubeDL.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index bae52e9c7..df2aebb59 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -553,20 +553,16 @@ class YoutubeDL(object): elif template_dict.get('width'): template_dict['resolution'] = '?x%d' % template_dict['width'] - restrict_filenames = self.params.get('restrictfilenames') - sanitize = lambda k, v: sanitize_filename( compat_str(v), - restricted=restrict_filenames, + restricted=self.params.get('restrictfilenames'), is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() if v is not None) template_dict = collections.defaultdict(lambda: 'NA', template_dict) - outtmpl = sanitize_filename( - self.params.get('outtmpl', DEFAULT_OUTTMPL), - restricted=restrict_filenames) + outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) tmpl = compat_expanduser(outtmpl) filename = tmpl % template_dict # Temporary fix for #4787 From d34e79492d1d9d9babf85af3737b90c4fe55eb42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 16:54:11 +0600 Subject: [PATCH 043/118] [twitch] Fix live streams (Closes #5158) --- youtube_dl/extractor/twitch.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index b058891bd..cbdaf9c7a 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -358,13 +358,12 @@ class TwitchStreamIE(TwitchBaseIE): 'p': random.randint(1000000, 10000000), 'player': 'twitchweb', 'segment_preference': '4', - 'sig': access_token['sig'], - 'token': access_token['token'], + 'sig': access_token['sig'].encode('utf-8'), + 'token': access_token['token'].encode('utf-8'), } - formats = self._extract_m3u8_formats( '%s/api/channel/hls/%s.m3u8?%s' - % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')), + % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)), channel_id, 'mp4') self._prefer_source(formats) From 1132eae56d0f693b5dc55529d5cfadf32b32700d Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Sun, 8 Mar 2015 13:54:01 +0200 Subject: [PATCH 044/118] [gazeta] Add new extractor (Closes #4222) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/gazeta.py | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 youtube_dl/extractor/gazeta.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index b489d5770..e2475a634 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -175,6 +175,7 @@ from .gameone import ( from .gamespot import GameSpotIE from .gamestar import GameStarIE from .gametrailers import GametrailersIE +from .gazeta import GazetaIE from .gdcvault import GDCVaultIE from .generic import GenericIE from .giantbomb import GiantBombIE diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py new file mode 100644 index 000000000..3f9e58061 --- /dev/null +++ b/youtube_dl/extractor/gazeta.py @@ -0,0 +1,35 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class GazetaIE(InfoExtractor): + _VALID_URL = r'(?Phttps?://(?:www\.)?gazeta\.ru/(?:(?P[^/]*)/)?video/(?:main/)?(?P[A-Za-z0-9-_]+)\.s?html)' + _TEST = { + 'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml', + 'md5': 'd49c9bdc6e5a7888f27475dc215ee789', + 'info_dict': { + 'id': '205566', + 'ext': 'mp4', + 'title': '«70–80 процентов гражданских в Донецке на грани голода»', + 'description': 'md5:38617526050bd17b234728e7f9620a71', + 'thumbnail': 're:^https?://.*\.jpg', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + display_id = mobj.group('id') + embed_url = '%s?p=embed' % mobj.group('url') + embed_page = self._download_webpage( + embed_url, display_id, 'Downloading embed page') + + video_id = self._search_regex( + r']*?class="eagleplayer"[^>]*?data-id="([^"]+)"', embed_page, 'video id') + + return self.url_result( + 'eagleplatform:gazeta.media.eagleplatform.com:%s' % video_id, 'EaglePlatform') From 28778d6bae2ff2a0ad57d2c5a694e22ac4ead749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 18:03:12 +0600 Subject: [PATCH 045/118] [pladform] Add extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/pladform.py | 86 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 youtube_dl/extractor/pladform.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index e2475a634..14172ca56 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -364,6 +364,7 @@ from .pbs import PBSIE from .phoenix import PhoenixIE from .photobucket import PhotobucketIE from .planetaplay import PlanetaPlayIE +from .pladform import PladformIE from .played import PlayedIE from .playfm import PlayFMIE from .playvid import PlayvidIE diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py new file mode 100644 index 000000000..926e368a2 --- /dev/null +++ b/youtube_dl/extractor/pladform.py @@ -0,0 +1,86 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + int_or_none, + xpath_text, +) + + +class PladformIE(InfoExtractor): + _VALID_URL = r'''(?x) + https?:// + (?: + (?: + out\.pladform\.ru/player| + static\.pladform\.ru/player\.swf + ) + \?.*\bvideoid=| + video\.pladform\.ru/catalog/video/videoid/ + ) + (?P\d+) + ''' + _TESTS = [{ + # http://muz-tv.ru/kinozal/view/7400/ + 'url': 'http://out.pladform.ru/player?pl=24822&videoid=100183293', + 'md5': '61f37b575dd27f1bb2e1854777fe31f4', + 'info_dict': { + 'id': '100183293', + 'ext': 'mp4', + 'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть', + 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 694, + 'age_limit': 0, + }, + }, { + 'url': 'http://static.pladform.ru/player.swf?pl=21469&videoid=100183293&vkcid=0', + 'only_matching': True, + }, { + 'url': 'http://video.pladform.ru/catalog/video/videoid/100183293/vkcid/0', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + video = self._download_xml( + 'http://out.pladform.ru/getVideo?pl=1&videoid=%s' % video_id, + video_id) + + if video.tag == 'error': + raise ExtractorError( + '%s returned error: %s' % (self.IE_NAME, video.text), + expected=True) + + formats = [{ + 'url': src.text, + 'format_id': src.get('quality'), + } for src in video.findall('./src')] + self._sort_formats(formats) + + webpage = self._download_webpage( + 'http://video.pladform.ru/catalog/video/videoid/%s' % video_id, + video_id) + + title = self._og_search_title(webpage, fatal=False) or xpath_text( + video, './/title', 'title', fatal=True) + description = self._search_regex( + r'\s*

([^<]+)

', webpage, 'description', fatal=False) + thumbnail = self._og_search_thumbnail(webpage) or xpath_text( + video, './/cover', 'cover') + + duration = int_or_none(xpath_text(video, './/time', 'duration')) + age_limit = int_or_none(xpath_text(video, './/age18', 'age limit')) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'age_limit': age_limit, + 'formats': formats, + } From f8388757269de8a5e87b9a507db55021b3090980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 18:07:10 +0600 Subject: [PATCH 046/118] [pladform] Add support for embeds --- youtube_dl/extractor/generic.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 013198b0d..4e6927b08 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -596,6 +596,19 @@ class GenericIE(InfoExtractor): 'view_count': int, }, }, + # Pladform embed + { + 'url': 'http://muz-tv.ru/kinozal/view/7400/', + 'info_dict': { + 'id': '100183293', + 'ext': 'mp4', + 'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть', + 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 694, + 'age_limit': 0, + }, + }, # RSS feed with enclosure { 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml', @@ -1193,6 +1206,12 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform') + # Look for Pladform embeds + mobj = re.search( + r']+src="(?Phttps?://out\.pladform\.ru/player\?.+?)"', webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'Pladform') + def check_video(vurl): if YoutubeIE.suitable(vurl): return True From 11101076a1b0eb36dd56b1b5aa1b7559f2d230b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 18:09:47 +0600 Subject: [PATCH 047/118] [pladform] Fix format quality sorting --- youtube_dl/extractor/pladform.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py index 926e368a2..abde34b94 100644 --- a/youtube_dl/extractor/pladform.py +++ b/youtube_dl/extractor/pladform.py @@ -6,6 +6,7 @@ from ..utils import ( ExtractorError, int_or_none, xpath_text, + qualities, ) @@ -55,9 +56,12 @@ class PladformIE(InfoExtractor): '%s returned error: %s' % (self.IE_NAME, video.text), expected=True) + quality = qualities(('ld', 'sd', 'hd')) + formats = [{ 'url': src.text, 'format_id': src.get('quality'), + 'quality': quality(src.get('quality')), } for src in video.findall('./src')] self._sort_formats(formats) From 24993e3b396407af9ccdf8dd893588df31aff8af Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Sun, 8 Mar 2015 14:08:45 +0200 Subject: [PATCH 048/118] [vidme] Fix view_count extraction and remove comment_count extraction (Fixes #5133) Comment counts seem to no longer be listed on vid.me --- youtube_dl/extractor/vidme.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py index 339c3d897..bd953fb4c 100644 --- a/youtube_dl/extractor/vidme.py +++ b/youtube_dl/extractor/vidme.py @@ -41,13 +41,10 @@ class VidmeIE(InfoExtractor): duration = float_or_none(self._html_search_regex( r'data-duration="([^"]+)"', webpage, 'duration', fatal=False)) view_count = str_to_int(self._html_search_regex( - r'\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False)) + r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False)) like_count = str_to_int(self._html_search_regex( r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">', webpage, 'like count', fatal=False)) - comment_count = str_to_int(self._html_search_regex( - r'class="js-comment-count"[^>]+data-count="([\d,\.\s]+)">', - webpage, 'comment count', fatal=False)) return { 'id': video_id, @@ -61,5 +58,4 @@ class VidmeIE(InfoExtractor): 'duration': duration, 'view_count': view_count, 'like_count': like_count, - 'comment_count': comment_count, } From 8b910bda0cdba859c1971a410aa5d2a51c6e4918 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Sun, 8 Mar 2015 14:28:53 +0200 Subject: [PATCH 049/118] [teamcoco] Fix extraction --- youtube_dl/extractor/teamcoco.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py index 5793dbc10..7cb06f351 100644 --- a/youtube_dl/extractor/teamcoco.py +++ b/youtube_dl/extractor/teamcoco.py @@ -53,10 +53,10 @@ class TeamcocoIE(InfoExtractor): embed = self._download_webpage( embed_url, video_id, 'Downloading embed page') - encoded_data = self._search_regex( - r'"preload"\s*:\s*"([^"]+)"', embed, 'encoded data') + player_data = self._parse_json(self._search_regex( + r'Y\.Ginger\.Module\.Player\((\{.*?\})\);', embed, 'player data'), video_id) data = self._parse_json( - base64.b64decode(encoded_data.encode('ascii')).decode('utf-8'), video_id) + base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id) formats = [] get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p']) From bdf6eee0aeed8df586569982eaaac04eecc0062d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 19:17:54 +0600 Subject: [PATCH 050/118] [gazeta] Extend _VALID_URL --- youtube_dl/extractor/gazeta.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py index 3f9e58061..ea32b621c 100644 --- a/youtube_dl/extractor/gazeta.py +++ b/youtube_dl/extractor/gazeta.py @@ -7,8 +7,8 @@ from .common import InfoExtractor class GazetaIE(InfoExtractor): - _VALID_URL = r'(?Phttps?://(?:www\.)?gazeta\.ru/(?:(?P[^/]*)/)?video/(?:main/)?(?P[A-Za-z0-9-_]+)\.s?html)' - _TEST = { + _VALID_URL = r'(?Phttps?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:(?:main|\d{4}/\d{2}/\d{2})/)?(?P[A-Za-z0-9-_.]+)\.s?html)' + _TESTS = [{ 'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml', 'md5': 'd49c9bdc6e5a7888f27475dc215ee789', 'info_dict': { @@ -18,7 +18,10 @@ class GazetaIE(InfoExtractor): 'description': 'md5:38617526050bd17b234728e7f9620a71', 'thumbnail': 're:^https?://.*\.jpg', }, - } + }, { + 'url': 'http://www.gazeta.ru/lifestyle/video/2015/03/08/master-klass_krasivoi_byt._delaem_vesennii_makiyazh.shtml', + 'only_matching': True, + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) From a2aaf4dbc6e5f5d345329b5a845111851453b6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 20:55:22 +0600 Subject: [PATCH 051/118] [utils] Add sanitize_path --- test/test_utils.py | 21 +++++++++++++++++++++ youtube_dl/utils.py | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index 64fad58ad..5ebb8d498 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -38,6 +38,7 @@ from youtube_dl.utils import ( parse_iso8601, read_batch_urls, sanitize_filename, + sanitize_path, shell_quote, smuggle_url, str_to_int, @@ -131,6 +132,26 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw') self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI') + def test_sanitize_path(self): + if sys.platform != 'win32': + return + + self.assertEqual(sanitize_path('abc'), 'abc') + self.assertEqual(sanitize_path('abc/def'), 'abc\\def') + self.assertEqual(sanitize_path('abc\\def'), 'abc\\def') + self.assertEqual(sanitize_path('abc|def'), 'abc#def') + self.assertEqual(sanitize_path('<>:"|?*'), '#######') + self.assertEqual(sanitize_path('C:/abc/def'), 'C:\\abc\\def') + self.assertEqual(sanitize_path('C?:/abc/def'), 'C##\\abc\\def') + + self.assertEqual(sanitize_path('\\\\?\\UNC\\ComputerName\\abc'), '\\\\?\\UNC\\ComputerName\\abc') + self.assertEqual(sanitize_path('\\\\?\\UNC/ComputerName/abc'), '\\\\?\\UNC\\ComputerName\\abc') + + self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc') + self.assertEqual(sanitize_path('\\\\?\\C:/abc'), '\\\\?\\C:\\abc') + self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f') + self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc') + def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7426e2a1f..0f49d602e 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -311,6 +311,24 @@ def sanitize_filename(s, restricted=False, is_id=False): return result +def sanitize_path(s): + """Sanitizes and normalizes path on Windows""" + if sys.platform != 'win32': + return s + drive, _ = os.path.splitdrive(s) + unc, _ = os.path.splitunc(s) + unc_or_drive = unc or drive + norm_path = os.path.normpath(remove_start(s, unc_or_drive)).split(os.path.sep) + if unc_or_drive: + norm_path.pop(0) + sanitized_path = [ + re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part) + for path_part in norm_path] + if unc_or_drive: + sanitized_path.insert(0, unc_or_drive + os.path.sep) + return os.path.join(*sanitized_path) + + def orderedSet(iterable): """ Remove all duplicates from the input iterable """ res = [] From d55de57b67bceca5d9116ddcc2ada2fa1957d89d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 20:56:28 +0600 Subject: [PATCH 052/118] [utils] Fix sanitize_open --- youtube_dl/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0f49d602e..e511232ca 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -252,15 +252,12 @@ def sanitize_open(filename, open_mode): raise # In case of error, try to remove win32 forbidden chars - alt_filename = os.path.join( - re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part) - for path_part in os.path.split(filename) - ) + alt_filename = sanitize_path(filename) if alt_filename == filename: raise else: # An exception here should be caught in the caller - stream = open(encodeFilename(filename), open_mode) + stream = open(encodeFilename(alt_filename), open_mode) return (stream, alt_filename) From 1bb5c511a551ba7c5d3179d2d3a124f0977e74b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 20:57:30 +0600 Subject: [PATCH 053/118] [YoutubeDL] Sanitize outtmpl as path --- youtube_dl/YoutubeDL.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index bae52e9c7..4b9151163 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -61,6 +61,7 @@ from .utils import ( render_table, SameFileError, sanitize_filename, + sanitize_path, std_headers, subtitles_filename, takewhile_inclusive, @@ -553,20 +554,16 @@ class YoutubeDL(object): elif template_dict.get('width'): template_dict['resolution'] = '?x%d' % template_dict['width'] - restrict_filenames = self.params.get('restrictfilenames') - sanitize = lambda k, v: sanitize_filename( compat_str(v), - restricted=restrict_filenames, + restricted=self.params.get('restrictfilenames'), is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() if v is not None) template_dict = collections.defaultdict(lambda: 'NA', template_dict) - outtmpl = sanitize_filename( - self.params.get('outtmpl', DEFAULT_OUTTMPL), - restricted=restrict_filenames) + outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL)) tmpl = compat_expanduser(outtmpl) filename = tmpl % template_dict # Temporary fix for #4787 From f18ef2d14463a13d80e967d1b18ece6a076f60fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 22:08:48 +0600 Subject: [PATCH 054/118] [utils] Disallow trailing dot in sanitize_path for a path part --- test/test_utils.py | 11 +++++++++++ youtube_dl/utils.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 5ebb8d498..28bda654e 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -152,6 +152,17 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f') self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc') + self.assertEqual( + sanitize_path('youtube/%(uploader)s/%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s'), + 'youtube\\%(uploader)s\\%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s') + + self.assertEqual( + sanitize_path('youtube/TheWreckingYard ./00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part'), + 'youtube\\TheWreckingYard #\\00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part') + self.assertEqual(sanitize_path('abc/def...'), 'abc\\def..#') + self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def') + self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#') + def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index e511232ca..d5597d514 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -319,7 +319,7 @@ def sanitize_path(s): if unc_or_drive: norm_path.pop(0) sanitized_path = [ - re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part) + re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part) for path_part in norm_path] if unc_or_drive: sanitized_path.insert(0, unc_or_drive + os.path.sep) From e5a11a2293069b1acfa5eb5d2694ad4082dd9755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 8 Mar 2015 22:09:42 +0600 Subject: [PATCH 055/118] [YoutubeDL] Sanitize path before creating non-existent paths (Closes #4324) --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4b9151163..bce7587fd 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1262,7 +1262,7 @@ class YoutubeDL(object): return try: - dn = os.path.dirname(encodeFilename(filename)) + dn = os.path.dirname(sanitize_path(encodeFilename(filename))) if dn and not os.path.exists(dn): os.makedirs(dn) except (OSError, IOError) as err: From 43d6280d0a03335ec5143383f15e2ca9a49f4046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 8 Mar 2015 18:25:11 +0100 Subject: [PATCH 056/118] [downloader/f4m] Fix use of base64 in python 3.2 (fixes #5132) b64decode needs a byte string, but on 3.4 it also accepts strings. --- youtube_dl/downloader/f4m.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py index 3dc796faa..4ab000d67 100644 --- a/youtube_dl/downloader/f4m.py +++ b/youtube_dl/downloader/f4m.py @@ -281,7 +281,7 @@ class F4mFD(FileDownloader): boot_info = self._get_bootstrap_from_url(bootstrap_url) else: bootstrap_url = None - bootstrap = base64.b64decode(node.text) + bootstrap = base64.b64decode(node.text.encode('ascii')) boot_info = read_bootstrap_info(bootstrap) return (boot_info, bootstrap_url) @@ -308,7 +308,7 @@ class F4mFD(FileDownloader): live = boot_info['live'] metadata_node = media.find(_add_ns('metadata')) if metadata_node is not None: - metadata = base64.b64decode(metadata_node.text) + metadata = base64.b64decode(metadata_node.text.encode('ascii')) else: metadata = None From 8bba753ccaab38a837efd5d334eff8f1b1c48fad Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Mar 2015 18:37:43 +0100 Subject: [PATCH 057/118] [options] Rename --dump-intermediate-pages to --dump-pages for consistence with --write-pages --- youtube_dl/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index a4ca8adc4..eefe008d5 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -563,7 +563,7 @@ def parseOpts(overrideArguments=None): action='store_true', dest='verbose', default=False, help='print various debugging information') verbosity.add_option( - '--dump-intermediate-pages', + '--dump-pages', '--dump-intermediate-pages', action='store_true', dest='dump_intermediate_pages', default=False, help='print downloaded pages to debug problems (very verbose)') verbosity.add_option( From cc08b11d16efd125d765cc1a69ac795a592f012c Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Sun, 8 Mar 2015 21:32:42 +0200 Subject: [PATCH 058/118] [adultswim] Improve video_info extraction (Fixes #5152) Look for video_info inside `slugged_video`, if slug is not found among collections. Also, simplify a bit. --- youtube_dl/extractor/adultswim.py | 44 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py index 34b8b0115..39335b827 100644 --- a/youtube_dl/extractor/adultswim.py +++ b/youtube_dl/extractor/adultswim.py @@ -2,13 +2,12 @@ from __future__ import unicode_literals import re -import json from .common import InfoExtractor from ..utils import ( ExtractorError, - xpath_text, float_or_none, + xpath_text, ) @@ -60,6 +59,24 @@ class AdultSwimIE(InfoExtractor): 'title': 'American Dad - Putting Francine Out of Business', 'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].' }, + }, { + 'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/', + 'playlist': [ + { + 'md5': '3e346a2ab0087d687a05e1e7f3b3e529', + 'info_dict': { + 'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0', + 'ext': 'flv', + 'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine', + 'description': 'Dr. Brule reports live from Wine Country with a special report on wines. \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n', + }, + } + ], + 'info_dict': { + 'id': 'sY3cMUR_TbuE4YmdjzbIcQ', + 'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine', + 'description': 'Dr. Brule reports live from Wine Country with a special report on wines. \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n', + }, }] @staticmethod @@ -80,6 +97,7 @@ class AdultSwimIE(InfoExtractor): for video in collection.get('videos'): if video.get('slug') == slug: return collection, video + return None, None def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -90,28 +108,30 @@ class AdultSwimIE(InfoExtractor): webpage = self._download_webpage(url, episode_path) # Extract the value of `bootstrappedData` from the Javascript in the page. - bootstrappedDataJS = self._search_regex(r'var bootstrappedData = ({.*});', webpage, episode_path) - - try: - bootstrappedData = json.loads(bootstrappedDataJS) - except ValueError as ve: - errmsg = '%s: Failed to parse JSON ' % episode_path - raise ExtractorError(errmsg, cause=ve) + bootstrapped_data = self._parse_json(self._search_regex( + r'var bootstrappedData = ({.*});', webpage, 'bootstraped data'), episode_path) # Downloading videos from a /videos/playlist/ URL needs to be handled differently. # NOTE: We are only downloading one video (the current one) not the playlist if is_playlist: - collections = bootstrappedData['playlists']['collections'] + collections = bootstrapped_data['playlists']['collections'] collection = self.find_collection_by_linkURL(collections, show_path) video_info = self.find_video_info(collection, episode_path) show_title = video_info['showTitle'] segment_ids = [video_info['videoPlaybackID']] else: - collections = bootstrappedData['show']['collections'] + collections = bootstrapped_data['show']['collections'] collection, video_info = self.find_collection_containing_video(collections, episode_path) - show = bootstrappedData['show'] + # Video wasn't found in the collections, let's try `slugged_video`. + if video_info is None: + if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path: + video_info = bootstrapped_data['slugged_video'] + else: + raise ExtractorError('Unable to find video info') + + show = bootstrapped_data['show'] show_title = show['title'] segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']] From dd7831fe94a0fb8270e7fa3699677c7476a5cd83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 9 Mar 2015 04:55:35 +0600 Subject: [PATCH 059/118] [breakcom] Process only play purpose media formats (Closes #5164) --- youtube_dl/extractor/breakcom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py index 4bcc897c9..809287d14 100644 --- a/youtube_dl/extractor/breakcom.py +++ b/youtube_dl/extractor/breakcom.py @@ -41,7 +41,7 @@ class BreakIE(InfoExtractor): 'tbr': media['bitRate'], 'width': media['width'], 'height': media['height'], - } for media in info['media']] + } for media in info['media'] if media.get('mediaPurpose') == 'play'] if not formats: formats.append({ From d475b3384cd372d89c35b9b1c26499f1e9cc1915 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Mar 2015 03:00:01 +0100 Subject: [PATCH 060/118] [README] Better bug reporting instructions Also address private emails which I get more and more these days. --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b9dd2cea..1a5a90c2e 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,18 @@ A note on the service that they don't host the infringing content, but just link Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content. +### How can I speed up work on my issue? + +(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do: + +First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel. + +Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues. + +If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so). + +Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`. + ### How can I detect whether a given URL is supported by youtube-dl? For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug. @@ -572,7 +584,9 @@ If your report is shorter than two lines, it is almost certainly missing some of For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information. -Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL. +If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/). + +**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL. ### Are you using the latest version? From f848215dfce232dfc6618c1cc49da44c3f7df75a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Mar 2015 03:02:03 +0100 Subject: [PATCH 061/118] release 2015.03.09 --- CONTRIBUTING.md | 4 +++- README.md | 2 +- docs/supportedsites.md | 5 +++++ youtube_dl/version.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 351229f21..588b15bde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,9 @@ If your report is shorter than two lines, it is almost certainly missing some of For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information. -Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL. +If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/). + +**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL. ### Are you using the latest version? diff --git a/README.md b/README.md index 1a5a90c2e..f41e7ecee 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ which means you can modify it, redistribute it or use it however you like. --no-progress do not print progress bar --console-title display progress in console titlebar -v, --verbose print various debugging information - --dump-intermediate-pages print downloaded pages to debug problems (very verbose) + --dump-pages print downloaded pages to debug problems (very verbose) --write-pages Write downloaded intermediary pages to files in the current directory to debug problems --print-traffic Display sent and read HTTP traffic -C, --call-home Contact the youtube-dl server for debugging. diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 062cb3d62..80a696ee3 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -117,6 +117,7 @@ - **DRTV** - **Dump** - **dvtv**: http://video.aktualne.cz/ + - **EaglePlatform** - **EbaumsWorld** - **EchoMsk** - **eHow** @@ -161,6 +162,7 @@ - **GameSpot** - **GameStar** - **Gametrailers** + - **Gazeta** - **GDCVault** - **generic**: Generic downloader that works on some sites - **GiantBomb** @@ -315,6 +317,7 @@ - **Ooyala** - **OpenFilm** - **orf:fm4**: radio FM4 + - **orf:iptv**: iptv.ORF.at - **orf:oe1**: Radio Österreich 1 - **orf:tvthek**: ORF TVthek - **parliamentlive.tv**: UK parliament videos @@ -322,10 +325,12 @@ - **PBS** - **Phoenix** - **Photobucket** + - **Pladform** - **PlanetaPlay** - **play.fm** - **played.to** - **Playvid** + - **Playwire** - **plus.google**: Google Plus - **pluzz.francetv.fr** - **podomatic** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 252933993..1f0c88a4d 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2015.03.03.1' +__version__ = '2015.03.09' From 4c6039385466e4be3a0e20459e75886052653de7 Mon Sep 17 00:00:00 2001 From: Mamay Alexander Date: Mon, 9 Mar 2015 19:06:49 +0600 Subject: [PATCH 062/118] [YandexMusic] Add new extractor --- youtube_dl/extractor/__init__.py | 5 ++ youtube_dl/extractor/yamusic.py | 104 +++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 youtube_dl/extractor/yamusic.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 14172ca56..2db938516 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -611,6 +611,11 @@ from .yahoo import ( YahooSearchIE, ) from .yam import YamIE +from .yamusic import ( + YandexMusicTrackIE, + YandexMusicAlbumIE, + YandexMusicPlaylistIE, +) from .yesjapan import YesJapanIE from .ynet import YnetIE from .youjizz import YouJizzIE diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yamusic.py new file mode 100644 index 000000000..5af6df89d --- /dev/null +++ b/youtube_dl/extractor/yamusic.py @@ -0,0 +1,104 @@ +# coding=utf-8 +from __future__ import unicode_literals + +import re +import hashlib +import time + +from .common import InfoExtractor + +class YandexMusicAlbumIE(InfoExtractor): + _VALID_URL = r'http://music.yandex.ru/album/(?P\d+)' + + def _get_track_url(self, storage_dir, track_id): + data = self._download_json('http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?requestId=2&nc=%d&action=getTrackSrc&p=download-info/%s/2.mp3' % (time.time(), storage_dir), track_id) + + hsh = hashlib.md5() + hsh.update('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']) + hash = hsh.hexdigest() + storage = storage_dir.split('.') + + return 'http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' % (data['host'], hash, data['ts'] + data['path'], storage[1]) + + def _get_album_id_and_data(self, url): + matched = re.match(self._VALID_URL, url) + id = matched.group('id') + + webpage = self._download_webpage(url, id) + data = self._parse_json( + self._search_regex( + r'var\s+Mu\s+=\s+(.+?);\s+<\/script>', webpage, 'player'), + id) + return id, data['pageData'] + + def _real_extract(self, url): + + id, data = self._get_album_id_and_data(url) + + entries = [] + + for track in data['volumes'][0]: + entries.append({ + 'id': track['id'], + 'ext': 'mp3', + 'url': self._get_track_url(track['storageDir'], track['id']), + 'title': track['artists'][0]['name'] + ' - ' + track['title'], + }) + + return { + '_type': 'playlist', + 'entries': entries, + 'id': id, + 'title': data['title'], + } + +class YandexMusicPlaylistIE(YandexMusicAlbumIE): + _VALID_URL = r'http://music.yandex.ru/users/(?P[^/]+)/playlists/(?P\d+)' + + def _real_extract(self, url): + id, data = self._get_album_id_and_data(url) + data = data['playlist'] + + entries = [] + + for track in data['tracks']: + entries.append({ + 'id': track['id'], + 'ext': 'mp3', + 'url': self._get_track_url(track['storageDir'], track['id']), + 'title': track['artists'][0]['name'] + ' - ' + track['title'], + }) + + return { + '_type': 'playlist', + 'entries': entries, + 'id': id, + 'title': data['title'], + } + +class YandexMusicTrackIE(YandexMusicAlbumIE): + _VALID_URL = r'http://music.yandex.ru/album/(?P\d+)/track/(?P\d+)' + _TEST = { + 'url': 'http://music.yandex.ru/album/540508/track/4878838', + 'info_dict': { + 'id': '4878838', + 'ext': 'mp3', + 'title': 'Carlo Ambrosio - Gypsy Eyes 1', + } + } + + def _real_extract(self, url): + + id, data = self._get_album_id_and_data(url) + + for track in data['volumes'][0]: + if track['id'] == id: + track_url = self._get_track_url(track['storageDir'], id) + break + + return { + 'id': id, + 'ext': 'mp3', + 'url': track_url, + 'title': track['artists'][0]['name'] + ' - ' + track['title'], + } From 47fe42e1ab3e182aacbc0c749b9a4f17f1b9dad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 9 Mar 2015 21:43:46 +0600 Subject: [PATCH 063/118] [yamusic] Improve, simplify, fix python3 issues and add tests --- youtube_dl/extractor/yamusic.py | 168 ++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 72 deletions(-) diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yamusic.py index 5af6df89d..e25c0be44 100644 --- a/youtube_dl/extractor/yamusic.py +++ b/youtube_dl/extractor/yamusic.py @@ -3,102 +3,126 @@ from __future__ import unicode_literals import re import hashlib -import time from .common import InfoExtractor +from ..compat import compat_str +from ..utils import ( + int_or_none, + float_or_none, +) -class YandexMusicAlbumIE(InfoExtractor): - _VALID_URL = r'http://music.yandex.ru/album/(?P\d+)' +class YandexMusicBaseIE(InfoExtractor): def _get_track_url(self, storage_dir, track_id): - data = self._download_json('http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?requestId=2&nc=%d&action=getTrackSrc&p=download-info/%s/2.mp3' % (time.time(), storage_dir), track_id) + data = self._download_json( + 'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s' + % storage_dir, + track_id, 'Downloading track location JSON') - hsh = hashlib.md5() - hsh.update('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']) - hash = hsh.hexdigest() + key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest() storage = storage_dir.split('.') - return 'http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' % (data['host'], hash, data['ts'] + data['path'], storage[1]) - - def _get_album_id_and_data(self, url): - matched = re.match(self._VALID_URL, url) - id = matched.group('id') - - webpage = self._download_webpage(url, id) - data = self._parse_json( - self._search_regex( - r'var\s+Mu\s+=\s+(.+?);\s+<\/script>', webpage, 'player'), - id) - return id, data['pageData'] - - def _real_extract(self, url): - - id, data = self._get_album_id_and_data(url) - - entries = [] - - for track in data['volumes'][0]: - entries.append({ - 'id': track['id'], - 'ext': 'mp3', - 'url': self._get_track_url(track['storageDir'], track['id']), - 'title': track['artists'][0]['name'] + ' - ' + track['title'], - }) + return ('http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' + % (data['host'], key, data['ts'] + data['path'], storage[1])) + def _get_track_info(self, track): return { - '_type': 'playlist', - 'entries': entries, - 'id': id, - 'title': data['title'], + 'id': track['id'], + 'ext': 'mp3', + 'url': self._get_track_url(track['storageDir'], track['id']), + 'title': '%s - %s' % (track['artists'][0]['name'], track['title']), + 'filesize': int_or_none(track.get('fileSize')), + 'duration': float_or_none(track.get('durationMs'), 1000), } -class YandexMusicPlaylistIE(YandexMusicAlbumIE): - _VALID_URL = r'http://music.yandex.ru/users/(?P[^/]+)/playlists/(?P\d+)' - def _real_extract(self, url): - id, data = self._get_album_id_and_data(url) - data = data['playlist'] +class YandexMusicTrackIE(YandexMusicBaseIE): + IE_NAME = 'yandexmusic:track' + IE_DESC = 'Яндекс.Музыка - Трек' + _VALID_URL = r'https?://music\.yandex\.ru/album/(?P\d+)/track/(?P\d+)' - entries = [] - - for track in data['tracks']: - entries.append({ - 'id': track['id'], - 'ext': 'mp3', - 'url': self._get_track_url(track['storageDir'], track['id']), - 'title': track['artists'][0]['name'] + ' - ' + track['title'], - }) - - return { - '_type': 'playlist', - 'entries': entries, - 'id': id, - 'title': data['title'], - } - -class YandexMusicTrackIE(YandexMusicAlbumIE): - _VALID_URL = r'http://music.yandex.ru/album/(?P\d+)/track/(?P\d+)' _TEST = { 'url': 'http://music.yandex.ru/album/540508/track/4878838', + 'md5': 'f496818aa2f60b6c0062980d2e00dc20', 'info_dict': { 'id': '4878838', 'ext': 'mp3', 'title': 'Carlo Ambrosio - Gypsy Eyes 1', + 'filesize': 4628061, + 'duration': 193.04, } } def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + album_id, track_id = mobj.group('album_id'), mobj.group('id') - id, data = self._get_album_id_and_data(url) + track = self._download_json( + 'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id), + track_id, 'Downloading track JSON')['track'] - for track in data['volumes'][0]: - if track['id'] == id: - track_url = self._get_track_url(track['storageDir'], id) - break + return self._get_track_info(track) + + +class YandexMusicAlbumIE(YandexMusicBaseIE): + IE_NAME = 'yandexmusic:album' + IE_DESC = 'Яндекс.Музыка - Альбом' + _VALID_URL = r'https?://music\.yandex\.ru/album/(?P\d+)' + + _TEST = { + 'url': 'http://music.yandex.ru/album/540508', + 'info_dict': { + 'id': '540508', + 'title': 'Carlo Ambrosio - Gypsy Soul (2009)', + }, + 'playlist_count': 50, + } + + def _real_extract(self, url): + album_id = self._match_id(url) + + album = self._download_json( + 'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id, + album_id, 'Downloading album JSON') + + entries = [self._get_track_info(track) for track in album['volumes'][0]] + + title = '%s - %s' % (album['artists'][0]['name'], album['title']) + year = album.get('year') + if year: + title += ' (%s)' % year + + return self.playlist_result(entries, compat_str(album['id']), title) + + +class YandexMusicPlaylistIE(YandexMusicBaseIE): + IE_NAME = 'yandexmusic:playlist' + IE_DESC = 'Яндекс.Музыка - Плейлист' + _VALID_URL = r'https?://music\.yandex\.ru/users/[^/]+/playlists/(?P\d+)' + + _TEST = { + 'url': 'http://music.yandex.ru/users/music.partners/playlists/1245', + 'info_dict': { + 'id': '1245', + 'title': 'Что слушают Enter Shikari', + 'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9', + }, + 'playlist_count': 6, + } + + def _real_extract(self, url): + playlist_id = self._match_id(url) + + webpage = self._download_webpage(url, playlist_id) + + playlist = self._parse_json( + self._search_regex( + r'var\s+Mu\s*=\s*({.+?});\s*', webpage, 'player'), + playlist_id)['pageData']['playlist'] + + entries = [self._get_track_info(track) for track in playlist['tracks']] + + return self.playlist_result( + entries, compat_str(playlist_id), + playlist['title'], playlist.get('description')) - return { - 'id': id, - 'ext': 'mp3', - 'url': track_url, - 'title': track['artists'][0]['name'] + ' - ' + track['title'], - } From d1e2e8f583524e837d452d37fa7175a55ebc80ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 9 Mar 2015 21:44:59 +0600 Subject: [PATCH 064/118] [yamusic] Rename to yandexmusic --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/{yamusic.py => yandexmusic.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename youtube_dl/extractor/{yamusic.py => yandexmusic.py} (100%) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 2db938516..20bc73dce 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -611,7 +611,7 @@ from .yahoo import ( YahooSearchIE, ) from .yam import YamIE -from .yamusic import ( +from .yandexmusic import ( YandexMusicTrackIE, YandexMusicAlbumIE, YandexMusicPlaylistIE, diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yandexmusic.py similarity index 100% rename from youtube_dl/extractor/yamusic.py rename to youtube_dl/extractor/yandexmusic.py From 41b2194f86c9440d2d095db3eaec90784703fa7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 9 Mar 2015 21:46:31 +0600 Subject: [PATCH 065/118] Credit @MamayAlexander for yandexmusic (#5168) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 4674a5af3..421df69a6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -113,3 +113,4 @@ Robin de Rooij Ryan Schmidt Leslie P. Polzer Duncan Keall +Alexander Mamay From dd77f14c641da26429ba32d2464563bf04e1741c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 9 Mar 2015 18:07:31 +0100 Subject: [PATCH 066/118] [yandexmusic] PEP8: remove blank line at the end of file --- youtube_dl/extractor/yandexmusic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py index e25c0be44..1ee8d1bc0 100644 --- a/youtube_dl/extractor/yandexmusic.py +++ b/youtube_dl/extractor/yandexmusic.py @@ -125,4 +125,3 @@ class YandexMusicPlaylistIE(YandexMusicBaseIE): return self.playlist_result( entries, compat_str(playlist_id), playlist['title'], playlist.get('description')) - From 937daef4a7725aeecc4b2ce0caa29c6f1aaf0b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 9 Mar 2015 18:12:41 +0100 Subject: [PATCH 067/118] [niconico] Use '_match_id' --- youtube_dl/extractor/niconico.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py index 4c1890416..7fb4e57df 100644 --- a/youtube_dl/extractor/niconico.py +++ b/youtube_dl/extractor/niconico.py @@ -41,7 +41,7 @@ class NiconicoIE(InfoExtractor): }, } - _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/((?:[a-z]{2})?[0-9]+)' + _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P(?:[a-z]{2})?[0-9]+)' _NETRC_MACHINE = 'niconico' # Determine whether the downloader used authentication to download video _AUTHENTICATED = False @@ -76,8 +76,7 @@ class NiconicoIE(InfoExtractor): return True def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1) + video_id = self._match_id(url) # Get video webpage. We are not actually interested in it, but need # the cookies in order to be able to download the info webpage From f5d8f58a170c9585e0913dbe1e50e0ed05afb698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 9 Mar 2015 18:17:22 +0100 Subject: [PATCH 068/118] [yandexmusic:album] Improve _VALID_URL to avoid matching tracks urls --- youtube_dl/extractor/yandexmusic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py index 1ee8d1bc0..b47aecb15 100644 --- a/youtube_dl/extractor/yandexmusic.py +++ b/youtube_dl/extractor/yandexmusic.py @@ -67,7 +67,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE): class YandexMusicAlbumIE(YandexMusicBaseIE): IE_NAME = 'yandexmusic:album' IE_DESC = 'Яндекс.Музыка - Альбом' - _VALID_URL = r'https?://music\.yandex\.ru/album/(?P\d+)' + _VALID_URL = r'https?://music\.yandex\.ru/album/(?P\d+)/?(\?|$)' _TEST = { 'url': 'http://music.yandex.ru/album/540508', From 2ebfeacabc7b74d03fa7cb096d8b9d1ecbdbb6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 10 Mar 2015 00:50:11 +0600 Subject: [PATCH 069/118] [utils] Keep dot and dotdot unmodified (Closes #5171) --- test/test_utils.py | 5 +++++ youtube_dl/utils.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 28bda654e..8f790bf0a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -163,6 +163,11 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def') self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#') + self.assertEqual(sanitize_path('../abc'), '..\\abc') + self.assertEqual(sanitize_path('../../abc'), '..\\..\\abc') + self.assertEqual(sanitize_path('./abc'), 'abc') + self.assertEqual(sanitize_path('./../abc'), '..\\abc') + def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d5597d514..c3135effc 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -319,7 +319,7 @@ def sanitize_path(s): if unc_or_drive: norm_path.pop(0) sanitized_path = [ - re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part) + path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part) for path_part in norm_path] if unc_or_drive: sanitized_path.insert(0, unc_or_drive + os.path.sep) From 614a7e1e230e095d9a11b59b20f4ff7462be8b21 Mon Sep 17 00:00:00 2001 From: "PishPosh.McGee" Date: Tue, 10 Mar 2015 15:22:46 -0500 Subject: [PATCH 070/118] Added subtitles for FunnyOrDie --- youtube_dl/extractor/funnyordie.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py index a49fc1151..f832ac5c3 100644 --- a/youtube_dl/extractor/funnyordie.py +++ b/youtube_dl/extractor/funnyordie.py @@ -50,7 +50,6 @@ class FunnyOrDieIE(InfoExtractor): bitrates.sort() formats = [] - for bitrate in bitrates: for link in links: formats.append({ @@ -59,6 +58,16 @@ class FunnyOrDieIE(InfoExtractor): 'vbr': bitrate, }) + subtitles={} + subtitle_matches=re.findall(r' Date: Tue, 10 Mar 2015 15:35:35 -0500 Subject: [PATCH 071/118] Update funnyordie.py --- youtube_dl/extractor/funnyordie.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py index f832ac5c3..0a22d99d2 100644 --- a/youtube_dl/extractor/funnyordie.py +++ b/youtube_dl/extractor/funnyordie.py @@ -65,8 +65,7 @@ class FunnyOrDieIE(InfoExtractor): if not sublang in subtitles.keys(): subtitles[sublang]=[] subext=suburl.split('/')[-1] - print subext - subtitles[sublang].append({'url': suburl,'ext': subext}) + subtitles[sublang].append({'url': 'http://www.funnyordie.com'+suburl,'ext': subext}) post_json = self._search_regex( r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details') From 32aaeca775c03ce784b8a30ea987c03a25a00e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 11 Mar 2015 20:34:32 +0600 Subject: [PATCH 072/118] [npo] Improve smooth stream skipping and set low preference for streams other than hds ans hls (Closes #5175) --- youtube_dl/extractor/npo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py index 9c01eb0af..557dffa46 100644 --- a/youtube_dl/extractor/npo.py +++ b/youtube_dl/extractor/npo.py @@ -219,7 +219,8 @@ class NPOLiveIE(NPOBaseIE): if streams: for stream in streams: stream_type = stream.get('type').lower() - if stream_type == 'ss': + # smooth streaming is not supported + if stream_type in ['ss', 'ms']: continue stream_info = self._download_json( 'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp' @@ -242,6 +243,7 @@ class NPOLiveIE(NPOBaseIE): else: formats.append({ 'url': stream_url, + 'preference': -10, }) self._sort_formats(formats) From c792b5011fe56af654d99378c1e77837475d7b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 11 Mar 2015 21:15:36 +0600 Subject: [PATCH 073/118] [ssa] Add extractor (Closes #5169) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/ssa.py | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 youtube_dl/extractor/ssa.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 20bc73dce..7adcc4dbf 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -460,6 +460,7 @@ from .sport5 import Sport5IE from .sportbox import SportBoxIE from .sportdeutschland import SportDeutschlandIE from .srmediathek import SRMediathekIE +from .ssa import SSAIE from .stanfordoc import StanfordOpenClassroomIE from .steam import SteamIE from .streamcloud import StreamcloudIE diff --git a/youtube_dl/extractor/ssa.py b/youtube_dl/extractor/ssa.py new file mode 100644 index 000000000..13101c714 --- /dev/null +++ b/youtube_dl/extractor/ssa.py @@ -0,0 +1,58 @@ +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + unescapeHTML, + parse_duration, +) + + +class SSAIE(InfoExtractor): + _VALID_URL = r'http://ssa\.nls\.uk/film/(?P\d+)' + _TEST = { + 'url': 'http://ssa.nls.uk/film/3561', + 'info_dict': { + 'id': '3561', + 'ext': 'flv', + 'title': 'SHETLAND WOOL', + 'description': 'md5:c5afca6871ad59b4271e7704fe50ab04', + 'duration': 900, + 'thumbnail': 're:^https?://.*\.jpg$', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage(url, video_id) + + streamer = self._search_regex( + r"'streamer'\s*,\S*'(rtmp[^']+)'", webpage, 'streamer') + play_path = self._search_regex( + r"'file'\s*,\s*'([^']+)'", webpage, 'file').rpartition('.')[0] + + def search_field(field_name, fatal=False): + return self._search_regex( + r'%s:\s*([^<]+)' % field_name, + webpage, 'title', fatal=fatal) + + title = unescapeHTML(search_field('Title', fatal=True)).strip('()[]') + description = unescapeHTML(search_field('Description')) + duration = parse_duration(search_field('Running time')) + thumbnail = self._search_regex( + r"'image'\s*,\s*'([^']+)'", webpage, 'thumbnails', fatal=False) + + return { + 'id': video_id, + 'url': streamer, + 'play_path': play_path, + 'ext': 'flv', + 'title': title, + 'description': description, + 'duration': duration, + 'thumbnail': thumbnail, + } From a3fbd188241b6151b024f6e3ea21c0edc60fae09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 11 Mar 2015 21:56:22 +0600 Subject: [PATCH 074/118] [funnyordie] Simplify subtitles --- youtube_dl/extractor/funnyordie.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py index 0a22d99d2..dd87257c4 100644 --- a/youtube_dl/extractor/funnyordie.py +++ b/youtube_dl/extractor/funnyordie.py @@ -58,14 +58,12 @@ class FunnyOrDieIE(InfoExtractor): 'vbr': bitrate, }) - subtitles={} - subtitle_matches=re.findall(r' Date: Wed, 11 Mar 2015 22:00:37 +0600 Subject: [PATCH 075/118] [funnyordie] Add subtitles test --- test/test_subtitles.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 3f2d8a2ba..891ee620b 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -26,6 +26,7 @@ from youtube_dl.extractor import ( VikiIE, ThePlatformIE, RTVEALaCartaIE, + FunnyOrDieIE, ) @@ -320,5 +321,17 @@ class TestRtveSubtitles(BaseTestSubtitles): self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca') +class TestFunnyOrDieSubtitles(BaseTestSubtitles): + url = 'http://www.funnyordie.com/videos/224829ff6d/judd-apatow-will-direct-your-vine' + IE = FunnyOrDieIE + + def test_allsubtitles(self): + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4') + + if __name__ == '__main__': unittest.main() From 3a77719c5ac60f7e634a7a8def82acb08ccc9ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 11 Mar 2015 17:38:35 +0100 Subject: [PATCH 076/118] Don't accept '-1' as format, 'all' is clearer --- youtube_dl/YoutubeDL.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index bce7587fd..be5b3c1ab 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1086,8 +1086,7 @@ class YoutubeDL(object): if req_format is None: req_format = 'best' formats_to_download = [] - # The -1 is for supporting YoutubeIE - if req_format in ('-1', 'all'): + if req_format == 'all': formats_to_download = formats else: for rfstr in req_format.split(','): From 71705fa70d3f8b410a25a9c1fb01c33dd5bb6026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 12 Mar 2015 21:56:56 +0600 Subject: [PATCH 077/118] [footyroom] Add extractor (Closes #5000) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/footyroom.py | 41 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 youtube_dl/extractor/footyroom.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 7adcc4dbf..7f9523c2b 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -151,6 +151,7 @@ from .fktv import ( ) from .flickr import FlickrIE from .folketinget import FolketingetIE +from .footyroom import FootyRoomIE from .fourtube import FourTubeIE from .foxgay import FoxgayIE from .foxnews import FoxNewsIE diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py new file mode 100644 index 000000000..2b4691ae8 --- /dev/null +++ b/youtube_dl/extractor/footyroom.py @@ -0,0 +1,41 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class FootyRoomIE(InfoExtractor): + _VALID_URL = r'http://footyroom\.com/(?P[^/]+)' + _TEST = { + 'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/', + 'info_dict': { + 'id': 'schalke-04-0-2-real-madrid-2015-02', + 'title': 'Schalke 04 0 – 2 Real Madrid', + }, + 'playlist_count': 3, + } + + def _real_extract(self, url): + playlist_id = self._match_id(url) + + webpage = self._download_webpage(url, playlist_id) + + playlist = self._parse_json( + self._search_regex( + r'VideoSelector\.load\((\[.+?\])\);', webpage, 'video selector'), + playlist_id) + + playlist_title = self._og_search_title(webpage) + + entries = [] + for video in playlist: + payload = video.get('payload') + if not payload: + continue + playwire_url = self._search_regex( + r'data-config="([^"]+)"', payload, + 'playwire url', default=None) + if playwire_url: + entries.append(self.url_result(playwire_url, 'Playwire')) + + return self.playlist_result(entries, playlist_id, playlist_title) From 1138491631c43c2f758d933675131f0bd8249920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 12 Mar 2015 21:59:46 +0600 Subject: [PATCH 078/118] [yam] Skip test --- youtube_dl/extractor/yam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py index 19ad74d04..19f8762ae 100644 --- a/youtube_dl/extractor/yam.py +++ b/youtube_dl/extractor/yam.py @@ -39,7 +39,8 @@ class YamIE(InfoExtractor): 'description': 'md5:11e2e405311633ace874f2e6226c8b17', 'uploader_id': '2323agoy', 'title': '20090412陽明山二子坪-1', - } + }, + 'skip': 'Video does not exist', }, { 'url': 'http://mymedia.yam.com/m/3598173', 'info_dict': { From d7d79106c7463b455ecc967c5f68b744c1375147 Mon Sep 17 00:00:00 2001 From: Leonardo Amaral Date: Thu, 12 Mar 2015 14:23:42 -0300 Subject: [PATCH 079/118] * Change globo.py flash ver to 17.0.0.132 - Chrome 42.0.2311.22 --- youtube_dl/extractor/globo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py index 29638a194..8a95793ca 100644 --- a/youtube_dl/extractor/globo.py +++ b/youtube_dl/extractor/globo.py @@ -20,7 +20,7 @@ class GloboIE(InfoExtractor): _VALID_URL = 'https?://.+?\.globo\.com/(?P.+)' _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist' - _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=2.9.9.50&resource_id=%s' + _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s' _VIDEOID_REGEXES = [ r'\bdata-video-id="(\d+)"', From b84037013edf25866772ba12db3b0c55f94a3e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 12 Mar 2015 18:45:00 +0100 Subject: [PATCH 080/118] [vimeo] Fix login (#3886) --- youtube_dl/extractor/vimeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index b84a83ba6..fc720e7e7 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -38,7 +38,7 @@ class VimeoBaseInfoExtractor(InfoExtractor): self.report_login() login_url = 'https://vimeo.com/log_in' webpage = self._download_webpage(login_url, None, False) - token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token') + token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token') data = urlencode_postdata({ 'email': username, 'password': password, From 3946864c8a752c068b188126013ddb747b53385d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 12 Mar 2015 19:08:16 +0100 Subject: [PATCH 081/118] [vimeo] Use https for all vimeo.com urls Unfortunately vimeopro.com doesn't support it yet. --- test/test_all_urls.py | 10 +++++----- youtube_dl/extractor/vimeo.py | 36 +++++++++++++++++------------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index e66264b4b..6ae168b7f 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -104,11 +104,11 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch(':tds', ['ComedyCentralShows']) def test_vimeo_matching(self): - self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel']) - self.assertMatch('http://vimeo.com/channels/31259', ['vimeo:channel']) - self.assertMatch('http://vimeo.com/channels/31259/53576664', ['vimeo']) - self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user']) - self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user']) + self.assertMatch('https://vimeo.com/channels/tributes', ['vimeo:channel']) + self.assertMatch('https://vimeo.com/channels/31259', ['vimeo:channel']) + self.assertMatch('https://vimeo.com/channels/31259/53576664', ['vimeo']) + self.assertMatch('https://vimeo.com/user7108434', ['vimeo:user']) + self.assertMatch('https://vimeo.com/user7108434/videos', ['vimeo:user']) self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review']) # https://github.com/rg3/youtube-dl/issues/1930 diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index fc720e7e7..bd09652cd 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -373,7 +373,7 @@ class VimeoIE(VimeoBaseInfoExtractor): for tt in text_tracks: subtitles[tt['lang']] = [{ 'ext': 'vtt', - 'url': 'http://vimeo.com' + tt['url'], + 'url': 'https://vimeo.com' + tt['url'], }] return { @@ -396,11 +396,11 @@ class VimeoIE(VimeoBaseInfoExtractor): class VimeoChannelIE(InfoExtractor): IE_NAME = 'vimeo:channel' - _VALID_URL = r'https?://vimeo\.com/channels/(?P[^/?#]+)/?(?:$|[?#])' + _VALID_URL = r'https://vimeo\.com/channels/(?P[^/?#]+)/?(?:$|[?#])' _MORE_PAGES_INDICATOR = r']+?title="(.*?)"' _TESTS = [{ - 'url': 'http://vimeo.com/channels/tributes', + 'url': 'https://vimeo.com/channels/tributes', 'info_dict': { 'id': 'tributes', 'title': 'Vimeo Tributes', @@ -459,7 +459,7 @@ class VimeoChannelIE(InfoExtractor): if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None: break - entries = [self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo') + entries = [self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo') for video_id in video_ids] return {'_type': 'playlist', 'id': list_id, @@ -470,15 +470,15 @@ class VimeoChannelIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) channel_id = mobj.group('id') - return self._extract_videos(channel_id, 'http://vimeo.com/channels/%s' % channel_id) + return self._extract_videos(channel_id, 'https://vimeo.com/channels/%s' % channel_id) class VimeoUserIE(VimeoChannelIE): IE_NAME = 'vimeo:user' - _VALID_URL = r'https?://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P[^/]+)(?:/videos|[#?]|$)' + _VALID_URL = r'https://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P[^/]+)(?:/videos|[#?]|$)' _TITLE_RE = r']+?class="user">([^<>]+?)' _TESTS = [{ - 'url': 'http://vimeo.com/nkistudio/videos', + 'url': 'https://vimeo.com/nkistudio/videos', 'info_dict': { 'title': 'Nki', 'id': 'nkistudio', @@ -489,7 +489,7 @@ class VimeoUserIE(VimeoChannelIE): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) name = mobj.group('name') - return self._extract_videos(name, 'http://vimeo.com/%s' % name) + return self._extract_videos(name, 'https://vimeo.com/%s' % name) class VimeoAlbumIE(VimeoChannelIE): @@ -526,9 +526,9 @@ class VimeoAlbumIE(VimeoChannelIE): class VimeoGroupsIE(VimeoAlbumIE): IE_NAME = 'vimeo:group' - _VALID_URL = r'(?:https?://)?vimeo\.com/groups/(?P[^/]+)' + _VALID_URL = r'https://vimeo\.com/groups/(?P[^/]+)' _TESTS = [{ - 'url': 'http://vimeo.com/groups/rolexawards', + 'url': 'https://vimeo.com/groups/rolexawards', 'info_dict': { 'id': 'rolexawards', 'title': 'Rolex Awards for Enterprise', @@ -542,13 +542,13 @@ class VimeoGroupsIE(VimeoAlbumIE): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) name = mobj.group('name') - return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name) + return self._extract_videos(name, 'https://vimeo.com/groups/%s' % name) class VimeoReviewIE(InfoExtractor): IE_NAME = 'vimeo:review' IE_DESC = 'Review pages on vimeo' - _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P[^/]+)' + _VALID_URL = r'https://vimeo\.com/[^/]+/review/(?P[^/]+)' _TESTS = [{ 'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d', 'md5': 'c507a72f780cacc12b2248bb4006d253', @@ -560,7 +560,7 @@ class VimeoReviewIE(InfoExtractor): } }, { 'note': 'video player needs Referer', - 'url': 'http://vimeo.com/user22258446/review/91613211/13f927e053', + 'url': 'https://vimeo.com/user22258446/review/91613211/13f927e053', 'md5': '6295fdab8f4bf6a002d058b2c6dce276', 'info_dict': { 'id': '91613211', @@ -582,11 +582,11 @@ class VimeoReviewIE(InfoExtractor): class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE): IE_NAME = 'vimeo:watchlater' IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)' - _VALID_URL = r'https?://vimeo\.com/home/watchlater|:vimeowatchlater' + _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater' _LOGIN_REQUIRED = True _TITLE_RE = r'href="/home/watchlater".*?>(.*?)<' _TESTS = [{ - 'url': 'http://vimeo.com/home/watchlater', + 'url': 'https://vimeo.com/home/watchlater', 'only_matching': True, }] @@ -606,7 +606,7 @@ class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE): class VimeoLikesIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?vimeo\.com/user(?P[0-9]+)/likes/?(?:$|[?#]|sort:)' + _VALID_URL = r'https://(?:www\.)?vimeo\.com/user(?P[0-9]+)/likes/?(?:$|[?#]|sort:)' IE_NAME = 'vimeo:likes' IE_DESC = 'Vimeo user likes' _TEST = { @@ -634,8 +634,8 @@ class VimeoLikesIE(InfoExtractor): description = self._html_search_meta('description', webpage) def _get_page(idx): - page_url = '%s//vimeo.com/user%s/likes/page:%d/sort:date' % ( - self.http_scheme(), user_id, idx + 1) + page_url = 'https://vimeo.com/user%s/likes/page:%d/sort:date' % ( + user_id, idx + 1) webpage = self._download_webpage( page_url, user_id, note='Downloading page %d/%d' % (idx + 1, page_count)) From 11984c7467184100ca4a61ae939a8c260480f42c Mon Sep 17 00:00:00 2001 From: "Devin J. Pohly" Date: Thu, 12 Mar 2015 15:43:13 -0400 Subject: [PATCH 082/118] [BeatportPro] Add new extractor This extractor is for Beatport's 2-minute, low-quality track previews only. To obtain an entire track, you obviously have to purchase and download it normally through the Beatport store! Possible future improvements: - Playlists for albums or other track-list pages - User login to play from My Beatport, Hold Bin, or Cart --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/beatportpro.py | 101 ++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 youtube_dl/extractor/beatportpro.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 7f9523c2b..ac765fdb8 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -37,6 +37,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE from .bbccouk import BBCCoUkIE from .beeg import BeegIE from .behindkink import BehindKinkIE +from .beatportpro import BeatportProIE from .bet import BetIE from .bild import BildIE from .bilibili import BiliBiliIE diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py new file mode 100644 index 000000000..21048b732 --- /dev/null +++ b/youtube_dl/extractor/beatportpro.py @@ -0,0 +1,101 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + +import re +import json + + +class BeatportProIE(InfoExtractor): + _VALID_URL = r'https?://pro\.beatport\.com/track/.*/(?P[0-9]+)' + _TESTS = [{ + 'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371', + 'md5': 'b3c34d8639a2f6a7f734382358478887', + 'info_dict': { + 'id': 5379371, + 'display-id': 'synesthesia-original-mix', + 'ext': 'mp4', + 'title': 'Froxic - Synesthesia (Original Mix)', + }, + }, { + 'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896', + 'md5': 'e44c3025dfa38c6577fbaeb43da43514', + 'info_dict': { + 'id': 3756896, + 'display-id': 'love-and-war-original-mix', + 'ext': 'mp3', + 'title': 'Wolfgang Gartner - Love & War (Original Mix)', + }, + }, { + 'url': 'https://pro.beatport.com/track/birds-original-mix/4991738', + 'md5': 'a1fd8e8046de3950fd039304c186c05f', + 'info_dict': { + 'id': 4991738, + 'display-id': 'birds-original-mix', + 'ext': 'mp4', + 'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)", + } + }] + + def _real_extract(self, url): + track_id = self._match_id(url) + webpage = self._download_webpage(url, track_id) + + # Extract "Playables" JSON information from the page + playables = self._search_regex(r'window\.Playables = ({.*?});', webpage, + 'playables info', flags=re.DOTALL) + playables = json.loads(playables) + + # Find first track with matching ID (always the first one listed?) + track = next(filter(lambda t: t['id'] == int(track_id), playables['tracks'])) + + # Construct title from artist(s), track name, and mix name + title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name'] + if track['mix']: + title += ' (' + track['mix'] + ')' + + # Get format information + formats = [] + for ext, info in track['preview'].items(): + if info['url'] is None: + continue + fmt = { + 'url': info['url'], + 'ext': ext, + 'format_id': ext, + 'vcodec': 'none', + } + if ext == 'mp3': + fmt['preference'] = 0 + fmt['acodec'] = 'mp3' + fmt['abr'] = 96 + fmt['asr'] = 44100 + elif ext == 'mp4': + fmt['preference'] = 1 + fmt['acodec'] = 'aac' + fmt['abr'] = 96 + fmt['asr'] = 44100 + formats += [fmt] + formats.sort(key=lambda f: f['preference']) + + # Get album art as thumbnails + imgs = [] + for name, info in track['images'].items(): + if name == 'dynamic' or info['url'] is None: + continue + img = { + 'id': name, + 'url': info['url'], + 'height': info['height'], + 'width': info['width'], + } + imgs += [img] + + return { + 'id': track['id'], + 'display-id': track['slug'], + 'title': title, + 'formats': formats, + 'thumbnails': imgs, + } From 65c5e044c7ab6d3140d30c98abda07785f2974c6 Mon Sep 17 00:00:00 2001 From: "Devin J. Pohly" Date: Thu, 12 Mar 2015 16:42:55 -0400 Subject: [PATCH 083/118] fix python2 --- youtube_dl/extractor/beatportpro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py index 21048b732..c3c70fb33 100644 --- a/youtube_dl/extractor/beatportpro.py +++ b/youtube_dl/extractor/beatportpro.py @@ -48,7 +48,7 @@ class BeatportProIE(InfoExtractor): playables = json.loads(playables) # Find first track with matching ID (always the first one listed?) - track = next(filter(lambda t: t['id'] == int(track_id), playables['tracks'])) + track = next(t for t in playables['tracks'] if t['id'] == int(track_id)) # Construct title from artist(s), track name, and mix name title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name'] From 054b99a33079dcc4755e46aaf588424b4bb12020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 12 Mar 2015 22:33:59 +0100 Subject: [PATCH 084/118] [jeuxvideo] Fix extraction (fixes #5190) --- youtube_dl/extractor/jeuxvideo.py | 36 ++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py index 8094cc2e4..d0720ff56 100644 --- a/youtube_dl/extractor/jeuxvideo.py +++ b/youtube_dl/extractor/jeuxvideo.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals -import json import re from .common import InfoExtractor @@ -15,10 +14,10 @@ class JeuxVideoIE(InfoExtractor): 'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm', 'md5': '046e491afb32a8aaac1f44dd4ddd54ee', 'info_dict': { - 'id': '5182', + 'id': '114765', 'ext': 'mp4', - 'title': 'GC 2013 : Tearaway nous présente ses papiers d\'identité', - 'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n', + 'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité', + 'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.', }, } @@ -26,26 +25,29 @@ class JeuxVideoIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) title = mobj.group(1) webpage = self._download_webpage(url, title) - xml_link = self._html_search_regex( - r'', + title = self._html_search_meta('name', webpage) + config_url = self._html_search_regex( + r'data-src="(/contenu/medias/video.php.*?)"', webpage, 'config URL') + config_url = 'http://www.jeuxvideo.com' + config_url video_id = self._search_regex( - r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml', - xml_link, 'video ID') + r'id=(\d+)', + config_url, 'video ID') - config = self._download_xml( - xml_link, title, 'Downloading XML config') - info_json = config.find('format.json').text - info = json.loads(info_json)['versions'][0] + config = self._download_json( + config_url, title, 'Downloading JSON config') - video_url = 'http://video720.jeuxvideo.com/' + info['file'] + formats = [{ + 'url': source['file'], + 'format_id': source['label'], + 'resolution': source['label'], + } for source in reversed(config['sources'])] return { 'id': video_id, - 'title': config.find('titre_video').text, - 'ext': 'mp4', - 'url': video_url, + 'title': title, + 'formats': formats, 'description': self._og_search_description(webpage), - 'thumbnail': config.find('image').text, + 'thumbnail': config.get('image'), } From 486dd09e0bd8032a3234363231c2b592f725cad6 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 13 Mar 2015 08:40:20 +0100 Subject: [PATCH 085/118] [YoutubeDL] Check for bytes instead of unicode output templates (#5192) Also adapt the embedding examples for those poor souls still using 2.x. --- README.md | 2 ++ youtube_dl/YoutubeDL.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index f41e7ecee..f4fbf0034 100644 --- a/README.md +++ b/README.md @@ -515,6 +515,7 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho From a Python program, you can embed youtube-dl in a more powerful fashion, like this: ```python +from __future__ import unicode_literals import youtube_dl ydl_opts = {} @@ -527,6 +528,7 @@ Most likely, you'll want to use various options. For a list of what can be done, Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file: ```python +from __future__ import unicode_literals import youtube_dl diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index be5b3c1ab..e779fc9a8 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -323,6 +323,11 @@ class YoutubeDL(object): 'Set the LC_ALL environment variable to fix this.') self.params['restrictfilenames'] = True + if isinstance(params.get('outtmpl'), bytes): + self.report_warning( + 'Parameter outtmpl is bytes, but should be a unicode string. ' + 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.') + if '%(stitle)s' in self.params.get('outtmpl', ''): self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') From fb7e68833cc67c23428462bf6c1306cbce207369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 20:51:44 +0600 Subject: [PATCH 086/118] [kanalplay] Add extractor (Closes #5188) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/kanalplay.py | 72 +++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 youtube_dl/extractor/kanalplay.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 7f9523c2b..5dc56e330 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -231,6 +231,7 @@ from .jove import JoveIE from .jukebox import JukeboxIE from .jpopsukitv import JpopsukiIE from .kaltura import KalturaIE +from .kanalplay import KanalPlayIE from .kankan import KankanIE from .karaoketv import KaraoketvIE from .keezmovies import KeezMoviesIE diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py new file mode 100644 index 000000000..869757ec0 --- /dev/null +++ b/youtube_dl/extractor/kanalplay.py @@ -0,0 +1,72 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + float_or_none, +) + + +class KanalPlayIE(InfoExtractor): + IE_DESC = 'Kanal 5/9/11 Play' + _VALID_URL = r'https?://(?:www\.)?kanal(?P5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P\d+)' + _TESTS = [{ + 'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277', + 'md5': '', + 'info_dict': { + 'id': '2609989', + 'ext': 'flv', + 'title': 'Saknar både dusch och avlopp', + 'description': 'md5:', + 'duration': 2636.36, + }, + }, { + 'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042', + 'only_matching': True, + }, { + 'url': 'http://www.kanal11play.se/#!/play/program/232835958/video/367135199', + 'only_matching': True, + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + channel_id = mobj.group('channel_id') + + video = self._download_json( + 'http://www.kanal%splay.se/api/getVideo?format=FLASH&videoId=%s' % (channel_id, video_id), + video_id) + + reasons_for_no_streams = video.get('reasonsForNoStreams') + if reasons_for_no_streams: + raise ExtractorError( + '%s returned error: %s' % (self.IE_NAME, '\n'.join(reasons_for_no_streams)), + expected=True) + + title = video['title'] + description = video.get('description') + duration = float_or_none(video.get('length'), 1000) + thumbnail = video.get('posterUrl') + + stream_base_url = video['streamBaseUrl'] + + formats = [{ + 'url': stream_base_url, + 'play_path': stream['source'], + 'ext': 'flv', + 'tbr': float_or_none(stream.get('bitrate'), 1000), + 'rtmp_real_time': True, + } for stream in video['streams']] + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'formats': formats, + } From 4a34f69ea6213ffcf5785b30a126b439ceef77e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 21:38:28 +0600 Subject: [PATCH 087/118] [extractor/common] Add subtitles timecode formatter --- youtube_dl/extractor/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index f9e8e2bad..e5245ec3f 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1062,6 +1062,9 @@ class InfoExtractor(object): def _get_automatic_captions(self, *args, **kwargs): raise NotImplementedError("This method must be implemented by subclasses") + def _subtitles_timecode(self, seconds): + return '%02d:%02d:%02d.%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000) + class SearchInfoExtractor(InfoExtractor): """ From 3f4327520c316043912ca9a8cd6f70d8121c3aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 21:39:29 +0600 Subject: [PATCH 088/118] [kanalplay] Extract subtitles --- youtube_dl/extractor/kanalplay.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py index 869757ec0..772d5aca4 100644 --- a/youtube_dl/extractor/kanalplay.py +++ b/youtube_dl/extractor/kanalplay.py @@ -31,6 +31,22 @@ class KanalPlayIE(InfoExtractor): 'only_matching': True, }] + def _fix_subtitles(self, subs): + return '\r\n\r\n'.join( + '%s\r\n%s --> %s\r\n%s' + % ( + num, + self._subtitles_timecode(item['startMillis'] / 1000.0), + self._subtitles_timecode(item['endMillis'] / 1000.0), + item['text'], + ) for num, item in enumerate(subs, 1)) + + def _get_subtitles(self, channel_id, video_id): + subs = self._download_json( + 'http://www.kanal%splay.se/api/subtitles/%s' % (channel_id, video_id), + video_id, 'Downloading subtitles JSON', fatal=False) + return {'se': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {} + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') @@ -62,6 +78,10 @@ class KanalPlayIE(InfoExtractor): } for stream in video['streams']] self._sort_formats(formats) + subtitles = {} + if video.get('hasSubtitle'): + subtitles = self.extract_subtitles(channel_id, video_id) + return { 'id': video_id, 'title': title, @@ -69,4 +89,5 @@ class KanalPlayIE(InfoExtractor): 'thumbnail': thumbnail, 'duration': duration, 'formats': formats, + 'subtitles': subtitles, } From 545315a9856743541f7f7b9b0ab42f2af9dd05c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 21:40:34 +0600 Subject: [PATCH 089/118] [nrk] Use generic subtitles timecode formatter --- youtube_dl/extractor/nrk.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py index 1e4cfa2e7..bff36f9d3 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dl/extractor/nrk.py @@ -149,9 +149,6 @@ class NRKTVIE(InfoExtractor): } ] - def _seconds2str(self, s): - return '%02d:%02d:%02d.%03d' % (s / 3600, (s % 3600) / 60, s % 60, (s % 1) * 1000) - def _debug_print(self, txt): if self._downloader.params.get('verbose', False): self.to_screen('[debug] %s' % txt) @@ -168,8 +165,8 @@ class NRKTVIE(InfoExtractor): for pos, p in enumerate(ps): begin = parse_duration(p.get('begin')) duration = parse_duration(p.get('dur')) - starttime = self._seconds2str(begin) - endtime = self._seconds2str(begin + duration) + starttime = self._subtitles_timecode(begin) + endtime = self._subtitles_timecode(begin + duration) srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text) return {lang: [ {'ext': 'ttml', 'url': url}, From 1b53778175e43e2bf2cb71885a760d96727ee837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 21:51:49 +0600 Subject: [PATCH 090/118] [beatenpro] Use generic format sort --- youtube_dl/extractor/beatportpro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py index c3c70fb33..bc201572e 100644 --- a/youtube_dl/extractor/beatportpro.py +++ b/youtube_dl/extractor/beatportpro.py @@ -77,7 +77,7 @@ class BeatportProIE(InfoExtractor): fmt['abr'] = 96 fmt['asr'] = 44100 formats += [fmt] - formats.sort(key=lambda f: f['preference']) + self._sort_formats(formats) # Get album art as thumbnails imgs = [] From 517bcca29925548f9b9b121beec1391ef3ecedec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 22:01:15 +0600 Subject: [PATCH 091/118] [beatenpro] Simplify and improve --- youtube_dl/extractor/beatportpro.py | 34 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py index bc201572e..69657cbde 100644 --- a/youtube_dl/extractor/beatportpro.py +++ b/youtube_dl/extractor/beatportpro.py @@ -1,14 +1,15 @@ # coding: utf-8 from __future__ import unicode_literals -from .common import InfoExtractor - import re import json +from .common import InfoExtractor +from ..utils import int_or_none + class BeatportProIE(InfoExtractor): - _VALID_URL = r'https?://pro\.beatport\.com/track/.*/(?P[0-9]+)' + _VALID_URL = r'https?://pro\.beatport\.com/track/.+/(?P[0-9]+)' _TESTS = [{ 'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371', 'md5': 'b3c34d8639a2f6a7f734382358478887', @@ -42,20 +43,17 @@ class BeatportProIE(InfoExtractor): track_id = self._match_id(url) webpage = self._download_webpage(url, track_id) - # Extract "Playables" JSON information from the page - playables = self._search_regex(r'window\.Playables = ({.*?});', webpage, - 'playables info', flags=re.DOTALL) + playables = self._search_regex( + r'window\.Playables\s*=\s*({.*?});', webpage, + 'playables info', flags=re.DOTALL) playables = json.loads(playables) - # Find first track with matching ID (always the first one listed?) track = next(t for t in playables['tracks'] if t['id'] == int(track_id)) - # Construct title from artist(s), track name, and mix name title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name'] if track['mix']: title += ' (' + track['mix'] + ')' - # Get format information formats = [] for ext, info in track['preview'].items(): if info['url'] is None: @@ -76,26 +74,26 @@ class BeatportProIE(InfoExtractor): fmt['acodec'] = 'aac' fmt['abr'] = 96 fmt['asr'] = 44100 - formats += [fmt] + formats.append(fmt) self._sort_formats(formats) - # Get album art as thumbnails - imgs = [] + images = [] for name, info in track['images'].items(): - if name == 'dynamic' or info['url'] is None: + image_url = info.get('url') + if name == 'dynamic' or not image_url: continue img = { 'id': name, - 'url': info['url'], - 'height': info['height'], - 'width': info['width'], + 'url': image_url, + 'height': int_or_none(info.get('height')), + 'width': int_or_none(info.get('width')), } - imgs += [img] + images.append(img) return { 'id': track['id'], 'display-id': track['slug'], 'title': title, 'formats': formats, - 'thumbnails': imgs, + 'thumbnails': images, } From ba1d4c04883cafb55e40734776d9d8ba2ef85582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 22:03:58 +0600 Subject: [PATCH 092/118] [beatenpro] Improve display_id --- youtube_dl/extractor/beatportpro.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py index 69657cbde..5c072b131 100644 --- a/youtube_dl/extractor/beatportpro.py +++ b/youtube_dl/extractor/beatportpro.py @@ -9,7 +9,7 @@ from ..utils import int_or_none class BeatportProIE(InfoExtractor): - _VALID_URL = r'https?://pro\.beatport\.com/track/.+/(?P[0-9]+)' + _VALID_URL = r'https?://pro\.beatport\.com/track/(?P[^/]+)/(?P[0-9]+)' _TESTS = [{ 'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371', 'md5': 'b3c34d8639a2f6a7f734382358478887', @@ -40,8 +40,11 @@ class BeatportProIE(InfoExtractor): }] def _real_extract(self, url): - track_id = self._match_id(url) - webpage = self._download_webpage(url, track_id) + mobj = re.match(self._VALID_URL, url) + track_id = mobj.group('id') + display_id = mobj.group('display_id') + + webpage = self._download_webpage(url, display_id) playables = self._search_regex( r'window\.Playables\s*=\s*({.*?});', webpage, @@ -92,7 +95,7 @@ class BeatportProIE(InfoExtractor): return { 'id': track['id'], - 'display-id': track['slug'], + 'display_id': track.get('slug') or display_id, 'title': title, 'formats': formats, 'thumbnails': images, From fcd877013e4a8f654c7778019055b57031492889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 22:11:56 +0600 Subject: [PATCH 093/118] [beatenpro] Simplify --- youtube_dl/extractor/beatportpro.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py index 5c072b131..12a7faa4f 100644 --- a/youtube_dl/extractor/beatportpro.py +++ b/youtube_dl/extractor/beatportpro.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals import re -import json from .common import InfoExtractor +from ..compat import compat_str from ..utils import int_or_none @@ -46,10 +46,11 @@ class BeatportProIE(InfoExtractor): webpage = self._download_webpage(url, display_id) - playables = self._search_regex( - r'window\.Playables\s*=\s*({.*?});', webpage, - 'playables info', flags=re.DOTALL) - playables = json.loads(playables) + playables = self._parse_json( + self._search_regex( + r'window\.Playables\s*=\s*({.+?});', webpage, + 'playables info', flags=re.DOTALL), + track_id) track = next(t for t in playables['tracks'] if t['id'] == int(track_id)) @@ -59,7 +60,7 @@ class BeatportProIE(InfoExtractor): formats = [] for ext, info in track['preview'].items(): - if info['url'] is None: + if not info['url']: continue fmt = { 'url': info['url'], @@ -85,16 +86,16 @@ class BeatportProIE(InfoExtractor): image_url = info.get('url') if name == 'dynamic' or not image_url: continue - img = { + image = { 'id': name, 'url': image_url, 'height': int_or_none(info.get('height')), 'width': int_or_none(info.get('width')), } - images.append(img) + images.append(image) return { - 'id': track['id'], + 'id': compat_str(track.get('id')) or track_id, 'display_id': track.get('slug') or display_id, 'title': title, 'formats': formats, From bba3fc7960c8cd6f0752c31c55ada804ba7e2ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 22:13:50 +0600 Subject: [PATCH 094/118] [beatenpro] Fix tests --- youtube_dl/extractor/beatportpro.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py index 12a7faa4f..3c7775d3e 100644 --- a/youtube_dl/extractor/beatportpro.py +++ b/youtube_dl/extractor/beatportpro.py @@ -14,8 +14,8 @@ class BeatportProIE(InfoExtractor): 'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371', 'md5': 'b3c34d8639a2f6a7f734382358478887', 'info_dict': { - 'id': 5379371, - 'display-id': 'synesthesia-original-mix', + 'id': '5379371', + 'display_id': 'synesthesia-original-mix', 'ext': 'mp4', 'title': 'Froxic - Synesthesia (Original Mix)', }, @@ -23,8 +23,8 @@ class BeatportProIE(InfoExtractor): 'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896', 'md5': 'e44c3025dfa38c6577fbaeb43da43514', 'info_dict': { - 'id': 3756896, - 'display-id': 'love-and-war-original-mix', + 'id': '3756896', + 'display_id': 'love-and-war-original-mix', 'ext': 'mp3', 'title': 'Wolfgang Gartner - Love & War (Original Mix)', }, @@ -32,8 +32,8 @@ class BeatportProIE(InfoExtractor): 'url': 'https://pro.beatport.com/track/birds-original-mix/4991738', 'md5': 'a1fd8e8046de3950fd039304c186c05f', 'info_dict': { - 'id': 4991738, - 'display-id': 'birds-original-mix', + 'id': '4991738', + 'display_id': 'birds-original-mix', 'ext': 'mp4', 'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)", } From 28c6411e4959e342ec8bc016eb8ca5dbcbdc5d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 13 Mar 2015 22:14:51 +0600 Subject: [PATCH 095/118] Credit @djpohly for BeatportPro (#5189) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 421df69a6..c10f03b98 100644 --- a/AUTHORS +++ b/AUTHORS @@ -114,3 +114,4 @@ Ryan Schmidt Leslie P. Polzer Duncan Keall Alexander Mamay +Devin J. Pohly From 13598940e3e019ce19eaf87ae872ea52b7cb8740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 14 Mar 2015 01:27:21 +0600 Subject: [PATCH 096/118] [kanalplay] Fix test --- youtube_dl/extractor/kanalplay.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py index 772d5aca4..2bb078036 100644 --- a/youtube_dl/extractor/kanalplay.py +++ b/youtube_dl/extractor/kanalplay.py @@ -15,14 +15,17 @@ class KanalPlayIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?kanal(?P5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P\d+)' _TESTS = [{ 'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277', - 'md5': '', 'info_dict': { - 'id': '2609989', + 'id': '3270012277', 'ext': 'flv', 'title': 'Saknar både dusch och avlopp', - 'description': 'md5:', + 'description': 'md5:6023a95832a06059832ae93bc3c7efb7', 'duration': 2636.36, }, + 'params': { + # rtmp download + 'skip_download': True, + } }, { 'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042', 'only_matching': True, From 3647136f24b332be81f9866ae5e2d596ca5c884a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 14 Mar 2015 02:12:11 +0600 Subject: [PATCH 097/118] [viewster] Add extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/viewster.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 youtube_dl/extractor/viewster.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 4682996cd..ad133603f 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -559,6 +559,7 @@ from .videoweed import VideoWeedIE from .vidme import VidmeIE from .vidzi import VidziIE from .vier import VierIE, VierVideosIE +from .viewster import ViewsterIE from .vimeo import ( VimeoIE, VimeoAlbumIE, diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py new file mode 100644 index 000000000..d7864d6a6 --- /dev/null +++ b/youtube_dl/extractor/viewster.py @@ -0,0 +1,53 @@ +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..compat import compat_urllib_request + + +class ViewsterIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P\d+-\d+-\d+)' + _TEST = { + 'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/', + 'md5': '8f9d94b282d80c42b378dffdbb11caf3', + 'info_dict': { + 'id': '1293-19341-000', + 'ext': 'flv', + 'title': "'Hout' (Wood)", + 'description': 'md5:925733185a9242ef96f436937683f33b', + }, + } + + _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01' + + def _real_extract(self, url): + video_id = self._match_id(url) + + request = compat_urllib_request.Request( + 'http://api.live.viewster.com/api/v1/movielink?movieid=%s&action=movierent&paymethod=fre&price=0¤cy=&language=en&subtitlelanguage=x&ischromecast=false' % video_id) + request.add_header('Accept', self._ACCEPT_HEADER) + + movie_link = self._download_json( + request, video_id, 'Downloading movie link JSON') + + formats = self._extract_f4m_formats( + movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id) + self._sort_formats(formats) + + request = compat_urllib_request.Request( + 'http://api.live.viewster.com/api/v1/movie/%s' % video_id) + request.add_header('Accept', self._ACCEPT_HEADER) + + movie = self._download_json( + request, video_id, 'Downloading movie metadata JSON') + + title = movie.get('title') or movie['original_title'] + description = movie.get('synopsis') + thumbnail = movie.get('large_artwork') or movie.get('artwork') + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'formats': formats, + } From 7be5a62ed71d574132b648864ce3b741635604f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 14 Mar 2015 03:18:04 +0600 Subject: [PATCH 098/118] [viewster] Improve extraction --- youtube_dl/extractor/viewster.py | 122 +++++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 23 deletions(-) diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py index d7864d6a6..1742e66f4 100644 --- a/youtube_dl/extractor/viewster.py +++ b/youtube_dl/extractor/viewster.py @@ -6,33 +6,75 @@ from ..compat import compat_urllib_request class ViewsterIE(InfoExtractor): _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P\d+-\d+-\d+)' - _TEST = { + _TESTS = [{ + # movielink, paymethod=fre 'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/', - 'md5': '8f9d94b282d80c42b378dffdbb11caf3', + 'playlist': [{ + 'md5': '8f9d94b282d80c42b378dffdbb11caf3', + 'info_dict': { + 'id': '1293-19341-000-movie', + 'ext': 'flv', + 'title': "'Hout' (Wood) - Movie", + }, + }], 'info_dict': { 'id': '1293-19341-000', - 'ext': 'flv', 'title': "'Hout' (Wood)", 'description': 'md5:925733185a9242ef96f436937683f33b', - }, - } + } + }, { + # movielink, paymethod=adv + 'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/', + 'playlist': [{ + 'md5': '77a005453ca7396cbe3d35c9bea30aef', + 'info_dict': { + 'id': '1140-11855-000-movie', + 'ext': 'flv', + 'title': "THE LISTENING PROJECT - Movie", + }, + }], + 'info_dict': { + 'id': '1140-11855-000', + 'title': "THE LISTENING PROJECT", + 'description': 'md5:714421ae9957e112e672551094bf3b08', + } + }, { + # direct links, no movielink + 'url': 'http://www.viewster.com/movie/1198-56411-000/sinister/', + 'playlist': [{ + 'md5': '0307b7eac6bfb21ab0577a71f6eebd8f', + 'info_dict': { + 'id': '1198-56411-000-trailer', + 'ext': 'mp4', + 'title': "Sinister - Trailer", + }, + }, { + 'md5': '80b9ee3ad69fb368f104cb5d9732ae95', + 'info_dict': { + 'id': '1198-56411-000-behind-scenes', + 'ext': 'mp4', + 'title': "Sinister - Behind Scenes", + }, + }, { + 'md5': '3b3ea897ecaa91fca57a8a94ac1b15c5', + 'info_dict': { + 'id': '1198-56411-000-scene-from-movie', + 'ext': 'mp4', + 'title': "Sinister - Scene from movie", + }, + }], + 'info_dict': { + 'id': '1198-56411-000', + 'title': "Sinister", + 'description': 'md5:014c40b0488848de9683566a42e33372', + } + }] _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01' def _real_extract(self, url): video_id = self._match_id(url) - request = compat_urllib_request.Request( - 'http://api.live.viewster.com/api/v1/movielink?movieid=%s&action=movierent&paymethod=fre&price=0¤cy=&language=en&subtitlelanguage=x&ischromecast=false' % video_id) - request.add_header('Accept', self._ACCEPT_HEADER) - - movie_link = self._download_json( - request, video_id, 'Downloading movie link JSON') - - formats = self._extract_f4m_formats( - movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id) - self._sort_formats(formats) - request = compat_urllib_request.Request( 'http://api.live.viewster.com/api/v1/movie/%s' % video_id) request.add_header('Accept', self._ACCEPT_HEADER) @@ -44,10 +86,44 @@ class ViewsterIE(InfoExtractor): description = movie.get('synopsis') thumbnail = movie.get('large_artwork') or movie.get('artwork') - return { - 'id': video_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'formats': formats, - } + entries = [] + for clip in movie['play_list']: + entry = None + + # movielink api + link_request = clip.get('link_request') + if link_request: + request = compat_urllib_request.Request( + 'http://api.live.viewster.com/api/v1/movielink?movieid=%(movieid)s&action=%(action)s&paymethod=%(paymethod)s&price=%(price)s¤cy=%(currency)s&language=%(language)s&subtitlelanguage=%(subtitlelanguage)s&ischromecast=%(ischromecast)s' + % link_request) + request.add_header('Accept', self._ACCEPT_HEADER) + + movie_link = self._download_json( + request, video_id, 'Downloading movie link JSON', fatal=False) + + if movie_link: + formats = self._extract_f4m_formats( + movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id) + self._sort_formats(formats) + entry = { + 'formats': formats, + } + + # direct link + clip_url = clip.get('clip_data', {}).get('url') + if clip_url: + entry = { + 'url': clip_url, + 'ext': 'mp4', + } + + if entry: + entry.update({ + 'id': '%s-%s' % (video_id, clip['canonical_title']), + 'title': '%s - %s' % (title, clip['title']), + }) + entries.append(entry) + + playlist = self.playlist_result(entries, video_id, title, description) + playlist['thumbnail'] = thumbnail + return playlist From 29171bc2d2f07a4eebb3b353fab989d7652a2083 Mon Sep 17 00:00:00 2001 From: Mamay Alexander Date: Sat, 14 Mar 2015 13:56:04 +0600 Subject: [PATCH 099/118] [yandexmusic] Site mirrors --- youtube_dl/extractor/yandexmusic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py index b47aecb15..f4c0f5702 100644 --- a/youtube_dl/extractor/yandexmusic.py +++ b/youtube_dl/extractor/yandexmusic.py @@ -39,7 +39,7 @@ class YandexMusicBaseIE(InfoExtractor): class YandexMusicTrackIE(YandexMusicBaseIE): IE_NAME = 'yandexmusic:track' IE_DESC = 'Яндекс.Музыка - Трек' - _VALID_URL = r'https?://music\.yandex\.ru/album/(?P\d+)/track/(?P\d+)' + _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P\d+)/track/(?P\d+)' _TEST = { 'url': 'http://music.yandex.ru/album/540508/track/4878838', @@ -67,7 +67,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE): class YandexMusicAlbumIE(YandexMusicBaseIE): IE_NAME = 'yandexmusic:album' IE_DESC = 'Яндекс.Музыка - Альбом' - _VALID_URL = r'https?://music\.yandex\.ru/album/(?P\d+)/?(\?|$)' + _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P\d+)/?(\?|$)' _TEST = { 'url': 'http://music.yandex.ru/album/540508', @@ -98,7 +98,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE): class YandexMusicPlaylistIE(YandexMusicBaseIE): IE_NAME = 'yandexmusic:playlist' IE_DESC = 'Яндекс.Музыка - Плейлист' - _VALID_URL = r'https?://music\.yandex\.ru/users/[^/]+/playlists/(?P\d+)' + _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P\d+)' _TEST = { 'url': 'http://music.yandex.ru/users/music.partners/playlists/1245', From 85741b998660bfb1564ca2c02e233db42059be05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 14 Mar 2015 15:52:06 +0600 Subject: [PATCH 100/118] [8tracks] Use predefined avg duration when duration is negative (Closes #5200) --- youtube_dl/extractor/eighttracks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index fb5dbbe2b..869ff72d6 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -117,6 +117,9 @@ class EightTracksIE(InfoExtractor): track_count = data['tracks_count'] duration = data['duration'] avg_song_duration = float(duration) / track_count + # duration is sometimes negative, use predefined avg duration + if avg_song_duration <= 0: + avg_song_duration = 300 first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) next_url = first_url entries = [] From 05be67e77dcf5c2a87d0549f6ae0f805ce38f758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 14 Mar 2015 15:54:23 +0600 Subject: [PATCH 101/118] [8tracks] Improve extraction --- youtube_dl/extractor/eighttracks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index 869ff72d6..49ec2be4c 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -108,9 +108,10 @@ class EightTracksIE(InfoExtractor): webpage = self._download_webpage(url, playlist_id) - json_like = self._search_regex( - r"(?s)PAGE.mix = (.*?);\n", webpage, 'trax information') - data = json.loads(json_like) + data = self._parse_json( + self._search_regex( + r"(?s)PAGE\.mix\s*=\s*({.+?});\n", webpage, 'trax information'), + playlist_id) session = str(random.randint(0, 1000000000)) mix_id = data['id'] From a7e01c438d2f7eb28a44056a1feb1b48f51e1096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 14 Mar 2015 15:55:21 +0600 Subject: [PATCH 102/118] [8tracks] Modernize --- youtube_dl/extractor/eighttracks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index 49ec2be4c..c84d16142 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -103,8 +103,7 @@ class EightTracksIE(InfoExtractor): } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') + playlist_id = self._match_id(url) webpage = self._download_webpage(url, playlist_id) From 9202b1b7878e2763a4406ebe2d59962202f6004b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 14 Mar 2015 12:04:49 +0100 Subject: [PATCH 103/118] [eighttracks] Remove unused import --- youtube_dl/extractor/eighttracks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index c84d16142..0b61ea0ba 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import json import random -import re from .common import InfoExtractor from ..compat import ( From 082b1155a36dc9b51424151f80860e52ee30b55e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 14 Mar 2015 12:06:01 +0100 Subject: [PATCH 104/118] [livestream] Extract all videos in events (fixes #5198) The webpage only contains the most recent ones, but if you scroll down more will appear. --- youtube_dl/extractor/livestream.py | 41 ++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py index 3642089f7..2467f8bdd 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dl/extractor/livestream.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import re import json +import itertools from .common import InfoExtractor from ..compat import ( @@ -40,6 +41,13 @@ class LivestreamIE(InfoExtractor): 'id': '2245590', }, 'playlist_mincount': 4, + }, { + 'url': 'http://new.livestream.com/chess24/tatasteelchess', + 'info_dict': { + 'title': 'Tata Steel Chess', + 'id': '3705884', + }, + 'playlist_mincount': 60, }, { 'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640', 'only_matching': True, @@ -117,6 +125,30 @@ class LivestreamIE(InfoExtractor): 'view_count': video_data.get('views'), } + def _extract_event(self, info): + event_id = compat_str(info['id']) + account = compat_str(info['owner_account_id']) + root_url = ( + 'https://new.livestream.com/api/accounts/{account}/events/{event}/' + 'feed.json'.format(account=account, event=event_id)) + + def _extract_videos(): + last_video = None + for i in itertools.count(1): + if last_video is None: + info_url = root_url + else: + info_url = '{root}?&id={id}&newer=-1&type=video'.format( + root=root_url, id=last_video) + videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data'] + videos_info = [v['data'] for v in videos_info if v['type'] == 'video'] + if not videos_info: + break + for v in videos_info: + yield self._extract_video_info(v) + last_video = videos_info[-1]['id'] + return self.playlist_result(_extract_videos(), event_id, info['full_name']) + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') @@ -144,14 +176,13 @@ class LivestreamIE(InfoExtractor): result = result and compat_str(vdata['data']['id']) == vid return result - videos = [self._extract_video_info(video_data['data']) - for video_data in info['feed']['data'] - if is_relevant(video_data, video_id)] if video_id is None: # This is an event page: - return self.playlist_result( - videos, '%s' % info['id'], info['full_name']) + return self._extract_event(info) else: + videos = [self._extract_video_info(video_data['data']) + for video_data in info['feed']['data'] + if is_relevant(video_data, video_id)] if not videos: raise ExtractorError('Cannot find video %s' % video_id) return videos[0] From e7db87f7000143341505cff812d1fa0371ac901e Mon Sep 17 00:00:00 2001 From: phiresky Date: Wed, 4 Mar 2015 22:33:56 +0100 Subject: [PATCH 105/118] Add metadata from title parser (Closes #5125) --- youtube_dl/__init__.py | 5 ++ youtube_dl/options.py | 9 ++++ youtube_dl/postprocessor/__init__.py | 2 + youtube_dl/postprocessor/ffmpeg.py | 10 +++- youtube_dl/postprocessor/metadatafromtitle.py | 48 +++++++++++++++++++ 5 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 youtube_dl/postprocessor/metadatafromtitle.py diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index a08ddd670..852b2fc3d 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -213,6 +213,11 @@ def _real_main(argv=None): # PostProcessors postprocessors = [] # Add the metadata pp first, the other pps will copy it + if opts.metafromtitle: + postprocessors.append({ + 'key': 'MetadataFromTitle', + 'titleformat': opts.metafromtitle + }) if opts.addmetadata: postprocessors.append({'key': 'FFmpegMetadata'}) if opts.extractaudio: diff --git a/youtube_dl/options.py b/youtube_dl/options.py index eefe008d5..be9402fdb 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -735,6 +735,15 @@ def parseOpts(overrideArguments=None): '--add-metadata', action='store_true', dest='addmetadata', default=False, help='write metadata to the video file') + postproc.add_option( + '--metadata-from-title', + metavar='FORMAT', dest='metafromtitle', + help='parse additional metadata like song title / artist from the video title. \n' + 'The format syntax is the same as --output, ' + 'the parsed parameters replace existing values.\n' + 'Additional templates: %(songtitle), %(album), %(artist). \n' + 'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like ' + '"Coldplay - Paradise"') postproc.add_option( '--xattrs', action='store_true', dest='xattrs', default=False, diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py index 708df3dd4..f39acadce 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dl/postprocessor/__init__.py @@ -15,6 +15,7 @@ from .ffmpeg import ( ) from .xattrpp import XAttrMetadataPP from .execafterdownload import ExecAfterDownloadPP +from .metadatafromtitle import MetadataFromTitlePP def get_postprocessor(key): @@ -34,5 +35,6 @@ __all__ = [ 'FFmpegPostProcessor', 'FFmpegSubtitlesConvertorPP', 'FFmpegVideoConvertorPP', + 'MetadataFromTitlePP', 'XAttrMetadataPP', ] diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 30094c2f3..a17113cbf 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -541,11 +541,15 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor): def run(self, info): metadata = {} - if info.get('title') is not None: + if info.get('songtitle') is not None: + metadata['title'] = info['songtitle'] + elif info.get('title') is not None: metadata['title'] = info['title'] if info.get('upload_date') is not None: metadata['date'] = info['upload_date'] - if info.get('uploader') is not None: + if info.get('artist') is not None: + metadata['artist'] = info['artist'] + elif info.get('uploader') is not None: metadata['artist'] = info['uploader'] elif info.get('uploader_id') is not None: metadata['artist'] = info['uploader_id'] @@ -554,6 +558,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor): metadata['comment'] = info['description'] if info.get('webpage_url') is not None: metadata['purl'] = info['webpage_url'] + if info.get('album') is not None: + metadata['album'] = info['album'] if not metadata: self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add') diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py new file mode 100644 index 000000000..4c9d3aafe --- /dev/null +++ b/youtube_dl/postprocessor/metadatafromtitle.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +import re + +from .common import PostProcessor +from ..utils import PostProcessingError + + +class MetadataFromTitlePPError(PostProcessingError): + pass + + +class MetadataFromTitlePP(PostProcessor): + def __init__(self, downloader, titleformat): + self._titleformat = titleformat + self._titleregex = self.fmtToRegex(titleformat) + + def fmtToRegex(self, fmt): + """ + Converts a string like + '%(title)s - %(artist)s' + to a regex like + '(?P.+)\ \-\ (?P<artist>.+)' + and a list of the named groups [title, artist] + """ + lastpos = 0 + regex = "" + groups = [] + # replace %(..)s with regex group and escape other string parts + for match in re.finditer(r'%\((\w+)\)s', fmt): + regex += re.escape(fmt[lastpos:match.start()]) + regex += r'(?P<' + match.group(1) + '>.+)' + lastpos = match.end() + if lastpos < len(fmt): + regex += re.escape(fmt[lastpos:len(fmt)]) + return regex + + def run(self, info): + title = info['title'] + match = re.match(self._titleregex, title) + if match is None: + raise MetadataFromTitlePPError('Could not interpret title of video as "%s"' % self._titleformat) + for attribute, value in match.groupdict().items(): + value = match.group(attribute) + info[attribute] = value + self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value) + + return True, info From 88cf6fb3685c4e012c9f574cbc5f1836c42fc06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 14 Mar 2015 19:55:42 +0100 Subject: [PATCH 106/118] [metadatafromtitle] Some improvements and cleanup * Remove the 'songtitle' field, 'title' can be used instead. * Remove newlines in the help text, for consistency with other options. * Add 'from __future__ import unicode_literals'. * Call '__init__' from the parent class. * Add test for the format_to_regex method --- test/test_postprocessors.py | 17 +++++++++++++++++ youtube_dl/options.py | 6 +++--- youtube_dl/postprocessor/ffmpeg.py | 4 +--- youtube_dl/postprocessor/metadatafromtitle.py | 9 ++++----- 4 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 test/test_postprocessors.py diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py new file mode 100644 index 000000000..addb69d6f --- /dev/null +++ b/test/test_postprocessors.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +from __future__ import unicode_literals + +# Allow direct execution +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl.postprocessor import MetadataFromTitlePP + + +class TestMetadataFromTitle(unittest.TestCase): + def test_format_to_regex(self): + pp = MetadataFromTitlePP(None, '%(title)s - %(artist)s') + self.assertEqual(pp._titleregex, '(?P<title>.+)\ \-\ (?P<artist>.+)') diff --git a/youtube_dl/options.py b/youtube_dl/options.py index be9402fdb..4e6e47d6f 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -738,10 +738,10 @@ def parseOpts(overrideArguments=None): postproc.add_option( '--metadata-from-title', metavar='FORMAT', dest='metafromtitle', - help='parse additional metadata like song title / artist from the video title. \n' + help='parse additional metadata like song title / artist from the video title. ' 'The format syntax is the same as --output, ' - 'the parsed parameters replace existing values.\n' - 'Additional templates: %(songtitle), %(album), %(artist). \n' + 'the parsed parameters replace existing values. ' + 'Additional templates: %(album), %(artist). ' 'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like ' '"Coldplay - Paradise"') postproc.add_option( diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index a17113cbf..b6f51cfd5 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -541,9 +541,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor): def run(self, info): metadata = {} - if info.get('songtitle') is not None: - metadata['title'] = info['songtitle'] - elif info.get('title') is not None: + if info.get('title') is not None: metadata['title'] = info['title'] if info.get('upload_date') is not None: metadata['date'] = info['upload_date'] diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py index 4c9d3aafe..5019433d3 100644 --- a/youtube_dl/postprocessor/metadatafromtitle.py +++ b/youtube_dl/postprocessor/metadatafromtitle.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import unicode_literals import re @@ -12,20 +12,19 @@ class MetadataFromTitlePPError(PostProcessingError): class MetadataFromTitlePP(PostProcessor): def __init__(self, downloader, titleformat): + super(MetadataFromTitlePP, self).__init__(downloader) self._titleformat = titleformat - self._titleregex = self.fmtToRegex(titleformat) + self._titleregex = self.format_to_regex(titleformat) - def fmtToRegex(self, fmt): + def format_to_regex(self, fmt): """ Converts a string like '%(title)s - %(artist)s' to a regex like '(?P<title>.+)\ \-\ (?P<artist>.+)' - and a list of the named groups [title, artist] """ lastpos = 0 regex = "" - groups = [] # replace %(..)s with regex group and escape other string parts for match in re.finditer(r'%\((\w+)\)s', fmt): regex += re.escape(fmt[lastpos:match.start()]) From 4d1652484f4a81b8d014159e680f7e20feae85e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 14 Mar 2015 20:25:37 +0100 Subject: [PATCH 107/118] [test/unicode_literals] Don't look into the .git and .tox directories The .tox directory contains python code that we can't control --- test/test_unicode_literals.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py index 7f816698e..6c1b7ec91 100644 --- a/test/test_unicode_literals.py +++ b/test/test_unicode_literals.py @@ -17,13 +17,22 @@ IGNORED_FILES = [ 'buildserver.py', ] +IGNORED_DIRS = [ + '.git', + '.tox', +] from test.helper import assertRegexpMatches class TestUnicodeLiterals(unittest.TestCase): def test_all_files(self): - for dirpath, _, filenames in os.walk(rootDir): + for dirpath, dirnames, filenames in os.walk(rootDir): + for ignore_dir in IGNORED_DIRS: + if ignore_dir in dirnames: + # If we remove the directory from dirnames os.walk won't + # recurse into it + dirnames.remove(ignore_dir) for basename in filenames: if not basename.endswith('.py'): continue From 8508557e77362bf1e5bcc919e89e6b4aa5d101ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 14 Mar 2015 20:51:42 +0100 Subject: [PATCH 108/118] [test/YoutubeDL] Use valid urls It failed on python 3.4 when building the http_headers field --- test/test_YoutubeDL.py | 46 ++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 055e42555..db8a47d2d 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -15,6 +15,8 @@ from youtube_dl import YoutubeDL from youtube_dl.extractor import YoutubeIE from youtube_dl.postprocessor.common import PostProcessor +TEST_URL = 'http://localhost/sample.mp4' + class YDL(FakeYDL): def __init__(self, *args, **kwargs): @@ -46,8 +48,8 @@ class TestFormatSelection(unittest.TestCase): ydl = YDL() ydl.params['prefer_free_formats'] = True formats = [ - {'ext': 'webm', 'height': 460, 'url': 'x'}, - {'ext': 'mp4', 'height': 460, 'url': 'y'}, + {'ext': 'webm', 'height': 460, 'url': TEST_URL}, + {'ext': 'mp4', 'height': 460, 'url': TEST_URL}, ] info_dict = _make_result(formats) yie = YoutubeIE(ydl) @@ -60,8 +62,8 @@ class TestFormatSelection(unittest.TestCase): ydl = YDL() ydl.params['prefer_free_formats'] = True formats = [ - {'ext': 'webm', 'height': 720, 'url': 'a'}, - {'ext': 'mp4', 'height': 1080, 'url': 'b'}, + {'ext': 'webm', 'height': 720, 'url': TEST_URL}, + {'ext': 'mp4', 'height': 1080, 'url': TEST_URL}, ] info_dict['formats'] = formats yie = YoutubeIE(ydl) @@ -74,9 +76,9 @@ class TestFormatSelection(unittest.TestCase): ydl = YDL() ydl.params['prefer_free_formats'] = False formats = [ - {'ext': 'webm', 'height': 720, 'url': '_'}, - {'ext': 'mp4', 'height': 720, 'url': '_'}, - {'ext': 'flv', 'height': 720, 'url': '_'}, + {'ext': 'webm', 'height': 720, 'url': TEST_URL}, + {'ext': 'mp4', 'height': 720, 'url': TEST_URL}, + {'ext': 'flv', 'height': 720, 'url': TEST_URL}, ] info_dict['formats'] = formats yie = YoutubeIE(ydl) @@ -88,8 +90,8 @@ class TestFormatSelection(unittest.TestCase): ydl = YDL() ydl.params['prefer_free_formats'] = False formats = [ - {'ext': 'flv', 'height': 720, 'url': '_'}, - {'ext': 'webm', 'height': 720, 'url': '_'}, + {'ext': 'flv', 'height': 720, 'url': TEST_URL}, + {'ext': 'webm', 'height': 720, 'url': TEST_URL}, ] info_dict['formats'] = formats yie = YoutubeIE(ydl) @@ -133,10 +135,10 @@ class TestFormatSelection(unittest.TestCase): def test_format_selection(self): formats = [ - {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': '_'}, - {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': '_'}, - {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': '_'}, - {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': '_'}, + {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL}, + {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL}, + {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL}, + {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL}, ] info_dict = _make_result(formats) @@ -167,10 +169,10 @@ class TestFormatSelection(unittest.TestCase): def test_format_selection_audio(self): formats = [ - {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': '_'}, - {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': '_'}, - {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': '_'}, - {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': '_'}, + {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL}, + {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL}, + {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL}, + {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL}, ] info_dict = _make_result(formats) @@ -185,8 +187,8 @@ class TestFormatSelection(unittest.TestCase): self.assertEqual(downloaded['format_id'], 'audio-low') formats = [ - {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': '_'}, - {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': '_'}, + {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL}, + {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL}, ] info_dict = _make_result(formats) @@ -228,9 +230,9 @@ class TestFormatSelection(unittest.TestCase): def test_format_selection_video(self): formats = [ - {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'}, - {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': '_'}, - {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': '_'}, + {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL}, + {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL}, + {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL}, ] info_dict = _make_result(formats) From 873383e9bd2496a31870e6456007463c1f873b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 14 Mar 2015 21:41:15 +0100 Subject: [PATCH 109/118] tox.ini: Run the same command as 'make offlinetest' by default --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ed01e3386..6fe7fc8f5 100644 --- a/tox.ini +++ b/tox.ini @@ -4,5 +4,8 @@ envlist = py26,py27,py33 deps = nose coverage -commands = nosetests --verbose {posargs:test} # --with-coverage --cover-package=youtube_dl --cover-html +defaultargs = test --exclude test_download.py --exclude test_age_restriction.py + --exclude test_subtitles.py --exclude test_write_annotations.py + --exclude test_youtube_lists.py +commands = nosetests --verbose {posargs:{[testenv]defaultargs}} # --with-coverage --cover-package=youtube_dl --cover-html # test.test_download:TestDownload.test_NowVideo From 7c42327e0e2f62a64771769e17cda33ea2a4a10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 14 Mar 2015 21:41:56 +0100 Subject: [PATCH 110/118] tox.ini: Add python 3.4 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6fe7fc8f5..00c6e00e3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33 +envlist = py26,py27,py33,py34 [testenv] deps = nose From 8f4cc22455f1c08f8e95f867f40c9f378bfe7a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sun, 15 Mar 2015 10:08:14 +0600 Subject: [PATCH 111/118] [aftenposten] Adapt to new URL format --- youtube_dl/extractor/aftenposten.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py index 2b257ede7..e15c015fb 100644 --- a/youtube_dl/extractor/aftenposten.py +++ b/youtube_dl/extractor/aftenposten.py @@ -14,10 +14,10 @@ from ..utils import ( class AftenpostenIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/([^/]+/)*(?P<id>[^/]+)-\d+\.html' + _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)' _TEST = { - 'url': 'http://www.aftenposten.no/webtv/serier-og-programmer/sweatshopenglish/TRAILER-SWEATSHOP---I-cant-take-any-more-7800835.html?paging=§ion=webtv_serierogprogrammer_sweatshop_sweatshopenglish', + 'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more', 'md5': 'fd828cd29774a729bf4d4425fe192972', 'info_dict': { 'id': '21039', @@ -30,12 +30,7 @@ class AftenpostenIE(InfoExtractor): } def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage(url, display_id) - - video_id = self._html_search_regex( - r'data-xs-id="(\d+)"', webpage, 'video id') + video_id = self._match_id(url) data = self._download_xml( 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id) From 613b2d9dc6eaf515898516c654f79cee8951a269 Mon Sep 17 00:00:00 2001 From: Kang Hyojun <admire9@gmail.com> Date: Sun, 15 Mar 2015 20:18:23 +0900 Subject: [PATCH 112/118] Fix mistyped docstring indent --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e779fc9a8..5a83bc956 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -635,7 +635,7 @@ class YoutubeDL(object): Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. extra_info is a dict containing the extra values to add to each result - ''' + ''' if ie_key: ies = [self.get_info_extractor(ie_key)] From ec1b9577ba31b7b078eec370680f84909b2e70e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sun, 15 Mar 2015 22:42:13 +0600 Subject: [PATCH 113/118] [cloudy] Fix key extraction (Closes #5211) --- youtube_dl/extractor/cloudy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py index abf8cc280..0fa720ee8 100644 --- a/youtube_dl/extractor/cloudy.py +++ b/youtube_dl/extractor/cloudy.py @@ -105,6 +105,7 @@ class CloudyIE(InfoExtractor): webpage = self._download_webpage(url, video_id) file_key = self._search_regex( - r'filekey\s*=\s*"([^"]+)"', webpage, 'file_key') + [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'], + webpage, 'file_key') return self._extract_video(video_host, video_id, file_key) From 39aa42ffbbe4df95ded3e70d4126808898a43f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Mon, 16 Mar 2015 00:21:38 +0600 Subject: [PATCH 114/118] [ard] Capture and output time restricted videos (Closes #5213) --- youtube_dl/extractor/ard.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py index 783b53e23..6a35ea463 100644 --- a/youtube_dl/extractor/ard.py +++ b/youtube_dl/extractor/ard.py @@ -50,6 +50,9 @@ class ARDMediathekIE(InfoExtractor): if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage: raise ExtractorError('Video %s is no longer available' % video_id, expected=True) + if 'Diese Sendung ist für Jugendliche unter 12 Jahren nicht geeignet. Der Clip ist deshalb nur von 20 bis 6 Uhr verfügbar.' in webpage: + raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True) + if re.search(r'[\?&]rss($|[=&])', url): doc = parse_xml(webpage) if doc.tag == 'rss': From 1de4ac1385bf0bedd753f41496055c53a14c20ca Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 15 Mar 2015 19:38:50 +0100 Subject: [PATCH 115/118] release 2015.03.15 --- README.md | 3 +++ docs/supportedsites.md | 8 ++++++++ youtube_dl/version.py | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f4fbf0034..4f9fc8174 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,9 @@ which means you can modify it, redistribute it or use it however you like. --embed-subs embed subtitles in the video (only for mp4 videos) --embed-thumbnail embed thumbnail in the audio as cover art --add-metadata write metadata to the video file + --metadata-from-title FORMAT parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed + parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s - + %(title)s" matches a title like "Coldplay - Paradise" --xattrs write metadata to the video file's xattrs (using dublin core and xdg standards) --fixup POLICY Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default; fix file if we can, warn otherwise) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 80a696ee3..d6a1e67c6 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -47,6 +47,7 @@ - **Bandcamp** - **Bandcamp:album** - **bbc.co.uk**: BBC iPlayer + - **BeatportPro** - **Beeg** - **BehindKink** - **Bet** @@ -145,6 +146,7 @@ - **Firstpost** - **Flickr** - **Folketinget**: Folketinget (ft.dk; Danish parliament) + - **FootyRoom** - **Foxgay** - **FoxNews** - **france2.fr:generation-quoi** @@ -213,6 +215,7 @@ - **jpopsuki.tv** - **Jukebox** - **Kaltura** + - **KanalPlay**: Kanal 5/9/11 Play - **Kankan** - **Karaoketv** - **keek** @@ -414,6 +417,7 @@ - **SportBox** - **SportDeutschland** - **SRMediathek**: Saarländischer Rundfunk + - **SSA** - **stanfordoc**: Stanford Open ClassRoom - **Steam** - **streamcloud.eu** @@ -510,6 +514,7 @@ - **Vidzi** - **vier** - **vier:videos** + - **Viewster** - **viki** - **vimeo** - **vimeo:album** @@ -556,6 +561,9 @@ - **XXXYMovies** - **Yahoo**: Yahoo screen and movies - **Yam** + - **yandexmusic:album**: Яндекс.Музыка - Альбом + - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист + - **yandexmusic:track**: Яндекс.Музыка - Трек - **YesJapan** - **Ynet** - **YouJizz** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 1f0c88a4d..7ed07c375 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2015.03.09' +__version__ = '2015.03.15' From d5b559393b793f21f6566321fa08a6bbbe6f524e Mon Sep 17 00:00:00 2001 From: Eduardo Ferro <eduardo.ferro.aldama@gmail.com> Date: Sun, 15 Mar 2015 21:45:14 +0100 Subject: [PATCH 116/118] [rtve] Add new extractor for rtve infantil --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/rtve.py | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ad133603f..1bb3e1a1c 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -405,7 +405,7 @@ from .rtlnow import RTLnowIE from .rtl2 import RTL2IE from .rtp import RTPIE from .rts import RTSIE -from .rtve import RTVEALaCartaIE, RTVELiveIE +from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE from .ruhd import RUHDIE from .rutube import ( RutubeIE, diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py index b42442d12..b701a322a 100644 --- a/youtube_dl/extractor/rtve.py +++ b/youtube_dl/extractor/rtve.py @@ -126,6 +126,49 @@ class RTVEALaCartaIE(InfoExtractor): (s['lang'], [{'ext': 'vtt', 'url': s['src']}]) for s in subs) +class RTVEInfantilIE(InfoExtractor): + IE_NAME = 'rtve.es:alacarta' + IE_DESC = 'RTVE a la carta' + _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_tittle>[^/]*)/(?P<id>[0-9]+)/' + + _TESTS = [{ + 'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/', + 'md5': '915319587b33720b8e0357caaa6617e6', + 'info_dict': { + 'id': '3040283', + 'ext': 'mp4', + 'title': 'Maneras de vivir', + 'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG', + 'duration': 357.958, + }, + },] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + short_tittle = mobj.group('short_tittle') + info = self._download_json( + 'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id, + video_id)['page']['items'][0] + + webpage = self._download_webpage(url, video_id) + vidplayer_id = self._search_regex( + r' id="vidplayer([0-9]+)"', webpage, 'internal video ID') + + png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id + png = self._download_webpage(png_url, video_id, 'Downloading url information') + video_url = _decrypt_url(png) + + return { + 'id': video_id, + 'ext': 'mp4', + 'title': info['title'], + 'url': video_url, + 'thumbnail': info.get('image'), + 'duration': float_or_none(info.get('duration'), scale=1000), + } + + class RTVELiveIE(InfoExtractor): IE_NAME = 'rtve.es:live' From b68eedba23cbb4cc419e3568055bfb86c5975054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 15 Mar 2015 22:17:40 +0100 Subject: [PATCH 117/118] [rtve.es:infantil] Minor fixes (closes #5214) --- youtube_dl/extractor/rtve.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py index b701a322a..13f071077 100644 --- a/youtube_dl/extractor/rtve.py +++ b/youtube_dl/extractor/rtve.py @@ -126,10 +126,11 @@ class RTVEALaCartaIE(InfoExtractor): (s['lang'], [{'ext': 'vtt', 'url': s['src']}]) for s in subs) + class RTVEInfantilIE(InfoExtractor): - IE_NAME = 'rtve.es:alacarta' - IE_DESC = 'RTVE a la carta' - _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_tittle>[^/]*)/(?P<id>[0-9]+)/' + IE_NAME = 'rtve.es:infantil' + IE_DESC = 'RTVE infantil' + _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_title>[^/]*)/(?P<id>[0-9]+)/' _TESTS = [{ 'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/', @@ -141,12 +142,10 @@ class RTVEInfantilIE(InfoExtractor): 'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG', 'duration': 357.958, }, - },] + }] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - short_tittle = mobj.group('short_tittle') + video_id = self._match_id(url) info = self._download_json( 'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id, video_id)['page']['items'][0] @@ -169,7 +168,6 @@ class RTVEInfantilIE(InfoExtractor): } - class RTVELiveIE(InfoExtractor): IE_NAME = 'rtve.es:live' IE_DESC = 'RTVE.es live streams' From 90183a46d85758c59814187d9c777e4237a66afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 15 Mar 2015 22:49:03 +0100 Subject: [PATCH 118/118] Credit @eferro for the rtve.es:infantil extractor (#5214) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index c10f03b98..872da6071 100644 --- a/AUTHORS +++ b/AUTHORS @@ -115,3 +115,4 @@ Leslie P. Polzer Duncan Keall Alexander Mamay Devin J. Pohly +Eduardo Ferro Aldama