From e3c1266f492d710e2acbf0d80f44f7f805eb5187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 18 May 2019 03:17:15 +0700 Subject: [PATCH 01/46] [extractor/common] Move workaround for applying first Set-Cookie header into a separate method --- youtube_dl/extractor/common.py | 23 +++++++++++++++++++++++ youtube_dl/extractor/vk.py | 22 +++------------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 69c3bc755..f994953bc 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -2817,6 +2817,29 @@ class InfoExtractor(object): self._downloader.cookiejar.add_cookie_header(req) return compat_cookies.SimpleCookie(req.get_header('Cookie')) + def _apply_first_set_cookie_header(self, url_handle, cookie): + # Some sites (e.g. [1-3]) may serve two cookies under the same name + # in Set-Cookie header and expect the first (old) one to be set rather + # than second (new). However, as of RFC6265 the newer one cookie + # should be set into cookie store what actually happens. + # We will workaround this issue by resetting the cookie to + # the first one manually. + # 1. https://new.vk.com/ + # 2. https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201 + # 3. https://learning.oreilly.com/ + for header, cookies in url_handle.headers.items(): + if header.lower() != 'set-cookie': + continue + if sys.version_info[0] >= 3: + cookies = cookies.encode('iso-8859-1') + cookies = cookies.decode('utf-8') + cookie_value = re.search( + r'%s=(.+?);.*?\b[Dd]omain=(.+?)(?:[,;]|$)' % cookie, cookies) + if cookie_value: + value, domain = cookie_value.groups() + self._set_cookie(domain, cookie, value) + break + def get_testcases(self, include_onlymatching=False): t = getattr(self, '_TEST', None) if t: diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py index b7ce2fb97..f57ed2288 100644 --- a/youtube_dl/extractor/vk.py +++ b/youtube_dl/extractor/vk.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import collections import re -import sys from .common import InfoExtractor from ..compat import compat_urlparse @@ -45,24 +44,9 @@ class VKBaseIE(InfoExtractor): 'pass': password.encode('cp1251'), }) - # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header - # and expects the first one to be set rather than second (see - # https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201). - # As of RFC6265 the newer one cookie should be set into cookie store - # what actually happens. - # We will workaround this VK issue by resetting the remixlhk cookie to - # the first one manually. - for header, cookies in url_handle.headers.items(): - if header.lower() != 'set-cookie': - continue - if sys.version_info[0] >= 3: - cookies = cookies.encode('iso-8859-1') - cookies = cookies.decode('utf-8') - remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies) - if remixlhk: - value, domain = remixlhk.groups() - self._set_cookie(domain, 'remixlhk', value) - break + # vk serves two same remixlhk cookies in Set-Cookie header and expects + # first one to be actually set + self._apply_first_set_cookie_header(url_handle, 'remixlhk') login_page = self._download_webpage( 'https://login.vk.com/?act=login', None, From a9e03736dfb2038d6a569e6305f4ac727a8ac71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 18 May 2019 03:23:40 +0700 Subject: [PATCH 02/46] [safari] Fix authentication (closes #21090) --- youtube_dl/extractor/safari.py | 87 ++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py index c0d32a1b9..8d4806794 100644 --- a/youtube_dl/extractor/safari.py +++ b/youtube_dl/extractor/safari.py @@ -1,15 +1,18 @@ # coding: utf-8 from __future__ import unicode_literals +import json import re from .common import InfoExtractor +from ..compat import ( + compat_parse_qs, + compat_str, + compat_urlparse, +) from ..utils import ( ExtractorError, - sanitized_Request, - std_headers, - urlencode_postdata, update_url_query, ) @@ -31,44 +34,52 @@ class SafariBaseIE(InfoExtractor): if username is None: return - headers = std_headers.copy() - if 'Referer' not in headers: - headers['Referer'] = self._LOGIN_URL + _, urlh = self._download_webpage_handle( + 'https://learning.oreilly.com/accounts/login-check/', None, + 'Downloading login page') - login_page = self._download_webpage( - self._LOGIN_URL, None, 'Downloading login form', headers=headers) + def is_logged(urlh): + return 'learning.oreilly.com/home/' in compat_str(urlh.geturl()) - def is_logged(webpage): - return any(re.search(p, webpage) for p in ( - r'href=["\']/accounts/logout/', r'>Sign Out<')) - - if is_logged(login_page): + if is_logged(urlh): self.LOGGED_IN = True return - csrf = self._html_search_regex( - r"name='csrfmiddlewaretoken'\s+value='([^']+)'", - login_page, 'csrf token') + redirect_url = compat_str(urlh.geturl()) + parsed_url = compat_urlparse.urlparse(redirect_url) + qs = compat_parse_qs(parsed_url.query) + next_uri = compat_urlparse.urljoin( + 'https://api.oreilly.com', qs['next'][0]) - login_form = { - 'csrfmiddlewaretoken': csrf, - 'email': username, - 'password1': password, - 'login': 'Sign In', - 'next': '', - } + auth, urlh = self._download_json_handle( + 'https://www.oreilly.com/member/auth/login/', None, 'Logging in', + data=json.dumps({ + 'email': username, + 'password': password, + 'redirect_uri': next_uri, + }).encode(), headers={ + 'Content-Type': 'application/json', + 'Referer': redirect_url, + }, expected_status=400) - request = sanitized_Request( - self._LOGIN_URL, urlencode_postdata(login_form), headers=headers) - login_page = self._download_webpage( - request, None, 'Logging in') - - if not is_logged(login_page): + credentials = auth.get('credentials') + if (not auth.get('logged_in') and not auth.get('redirect_uri') + and credentials): raise ExtractorError( - 'Login failed; make sure your credentials are correct and try again.', - expected=True) + 'Unable to login: %s' % credentials, expected=True) - self.LOGGED_IN = True + # oreilly serves two same groot_sessionid cookies in Set-Cookie header + # and expects first one to be actually set + self._apply_first_set_cookie_header(urlh, 'groot_sessionid') + + _, urlh = self._download_webpage_handle( + auth.get('redirect_uri') or next_uri, None, 'Completing login',) + + if is_logged(urlh): + self.LOGGED_IN = True + return + + raise ExtractorError('Unable to log in') class SafariIE(SafariBaseIE): @@ -76,7 +87,7 @@ class SafariIE(SafariBaseIE): IE_DESC = 'safaribooksonline.com online video' _VALID_URL = r'''(?x) https?:// - (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/ + (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/ (?: library/view/[^/]+/(?P[^/]+)/(?P[^/?\#&]+)\.html| videos/[^/]+/[^/]+/(?P[^-]+-[^/?\#&]+) @@ -107,6 +118,9 @@ class SafariIE(SafariBaseIE): }, { 'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro', 'only_matching': True, + }, { + 'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/00_SeriesIntro.html', + 'only_matching': True, }] _PARTNER_ID = '1926081' @@ -163,7 +177,7 @@ class SafariIE(SafariBaseIE): class SafariApiIE(SafariBaseIE): IE_NAME = 'safari:api' - _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/api/v1/book/(?P[^/]+)/chapter(?:-content)?/(?P[^/?#&]+)\.html' + _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/api/v1/book/(?P[^/]+)/chapter(?:-content)?/(?P[^/?#&]+)\.html' _TESTS = [{ 'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html', @@ -188,7 +202,7 @@ class SafariCourseIE(SafariBaseIE): _VALID_URL = r'''(?x) https?:// (?: - (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/ + (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/ (?: library/view/[^/]+| api/v1/book| @@ -219,6 +233,9 @@ class SafariCourseIE(SafariBaseIE): }, { 'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838', 'only_matching': True, + }, { + 'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/', + 'only_matching': True, }] @classmethod From ce2fe4c01cceef4b636995275b573baf51587fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 20 May 2019 23:23:18 +0700 Subject: [PATCH 03/46] [extractor/common] Add doc string for _apply_first_set_cookie_header --- youtube_dl/extractor/common.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index f994953bc..937237b3f 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -2818,15 +2818,19 @@ class InfoExtractor(object): return compat_cookies.SimpleCookie(req.get_header('Cookie')) def _apply_first_set_cookie_header(self, url_handle, cookie): - # Some sites (e.g. [1-3]) may serve two cookies under the same name - # in Set-Cookie header and expect the first (old) one to be set rather - # than second (new). However, as of RFC6265 the newer one cookie - # should be set into cookie store what actually happens. - # We will workaround this issue by resetting the cookie to - # the first one manually. - # 1. https://new.vk.com/ - # 2. https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201 - # 3. https://learning.oreilly.com/ + """ + Apply first Set-Cookie header instead of the last. Experimental. + + Some sites (e.g. [1-3]) may serve two cookies under the same name + in Set-Cookie header and expect the first (old) one to be set rather + than second (new). However, as of RFC6265 the newer one cookie + should be set into cookie store what actually happens. + We will workaround this issue by resetting the cookie to + the first one manually. + 1. https://new.vk.com/ + 2. https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201 + 3. https://learning.oreilly.com/ + """ for header, cookies in url_handle.headers.items(): if header.lower() != 'set-cookie': continue From 42c971341b804b758d12b7a85547be05160f1b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 20 May 2019 23:24:27 +0700 Subject: [PATCH 04/46] [ChangeLog] Actualize [ci skip] --- ChangeLog | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ChangeLog b/ChangeLog index 13cb6288d..eba7202dd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,22 @@ +version + +Core ++ [extractor/common] Move workaround for applying first Set-Cookie header + into a separate _apply_first_set_cookie_header method + +Extractors +* [safari] Fix authentication (#21090) +* [vk] Use _apply_first_set_cookie_header +* [vrt] Fix extraction (#20527) ++ [canvas] Add support for vrtnieuws and sporza site ids and extract + AES HLS formats ++ [vrv] Extract captions (#19238) +* [tele5] Improve video id extraction +* [tele5] Relax URL regular expression (#21020, #21063) +* [svtplay] Update API URL (#21075) ++ [yahoo:gyao] Add X-User-Agent header to dam proxy requests (#21071) + + version 2019.05.11 Core From 6ab30ff50bf6bd0585927cb73c7421bef184f87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 20 May 2019 23:29:49 +0700 Subject: [PATCH 05/46] release 2019.05.20 --- .github/ISSUE_TEMPLATE/1_broken_site.md | 6 +++--- .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++-- .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++-- .github/ISSUE_TEMPLATE/4_bug_report.md | 6 +++--- .github/ISSUE_TEMPLATE/5_feature_request.md | 4 ++-- ChangeLog | 2 +- docs/supportedsites.md | 2 +- youtube_dl/version.py | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index 6b931b3cf..dc303946e 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2019.05.11** +- [ ] I've verified that I'm running youtube-dl version **2019.05.20** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.05.11 + [debug] youtube-dl version 2019.05.20 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index f2dc784a2..46e143c8a 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2019.05.11** +- [ ] I've verified that I'm running youtube-dl version **2019.05.20** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index 39a0af13f..bc6c4694b 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2019.05.11** +- [ ] I've verified that I'm running youtube-dl version **2019.05.20** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index 139f36ab8..bcc51f986 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2019.05.11** +- [ ] I've verified that I'm running youtube-dl version **2019.05.20** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.05.11 + [debug] youtube-dl version 2019.05.20 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index d60da6db9..c8d16960e 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2019.05.11** +- [ ] I've verified that I'm running youtube-dl version **2019.05.20** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/ChangeLog b/ChangeLog index eba7202dd..3babb6f48 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version +version 2019.05.20 Core + [extractor/common] Move workaround for applying first Set-Cookie header diff --git a/docs/supportedsites.md b/docs/supportedsites.md index a8a9224cb..404a2f0a4 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1071,7 +1071,7 @@ - **VoxMediaVolume** - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **Vrak** - - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be + - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza - **VrtNU**: VrtNU.be - **vrv** - **vrv:series** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index e63527dbb..8df77378b 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.05.11' +__version__ = '2019.05.20' From 0e6f914b3b40ef2ca78d82051a194faaad64dd9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B6pfl?= Date: Wed, 13 Feb 2019 16:29:43 +0100 Subject: [PATCH 06/46] [vivo] Fix extraction (closes #18906) --- youtube_dl/extractor/shared.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py index 931a0f70e..eade8fd9e 100644 --- a/youtube_dl/extractor/shared.py +++ b/youtube_dl/extractor/shared.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..compat import compat_b64decode from ..utils import ( @@ -7,6 +9,7 @@ from ..utils import ( int_or_none, url_or_none, urlencode_postdata, + unescapeHTML, ) @@ -22,8 +25,7 @@ class SharedBaseIE(InfoExtractor): video_url = self._extract_video_url(webpage, video_id, url) - title = compat_b64decode(self._html_search_meta( - 'full:title', webpage, 'title')).decode('utf-8') + title = self._extract_title(webpage) filesize = int_or_none(self._html_search_meta( 'full:size', webpage, 'file size', fatal=False)) @@ -35,6 +37,10 @@ class SharedBaseIE(InfoExtractor): 'title': title, } + def _extract_title(self, webpage): + return compat_b64decode(self._html_search_meta( + 'full:title', webpage, 'title')).decode('utf-8') + class SharedIE(SharedBaseIE): IE_DESC = 'shared.sx' @@ -86,6 +92,14 @@ class VivoIE(SharedBaseIE): }, } + def _extract_title(self, webpage): + data_title = self._search_regex( + r'data-name\s*=\s*(["\'])(?P(?:(?!\1).)+)\1', webpage, + 'title', default=None, group='title') + if data_title: + return unescapeHTML(re.sub(r"\.[a-z0-9]{3,4}$", "", data_title)) + return self._og_search_title(webpage) + def _extract_video_url(self, webpage, video_id, *args): def decode_url(encoded_url): return compat_b64decode(encoded_url).decode('utf-8') From e438e8146965d2c650c1575dc97809bcc9504f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 23 May 2019 03:04:58 +0700 Subject: [PATCH 07/46] [vivo] Improve extraction (closes #19217) --- youtube_dl/extractor/shared.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py index eade8fd9e..ff575f592 100644 --- a/youtube_dl/extractor/shared.py +++ b/youtube_dl/extractor/shared.py @@ -1,15 +1,15 @@ from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..compat import compat_b64decode from ..utils import ( + determine_ext, ExtractorError, int_or_none, + KNOWN_EXTENSIONS, + parse_filesize, url_or_none, urlencode_postdata, - unescapeHTML, ) @@ -26,8 +26,7 @@ class SharedBaseIE(InfoExtractor): video_url = self._extract_video_url(webpage, video_id, url) title = self._extract_title(webpage) - filesize = int_or_none(self._html_search_meta( - 'full:size', webpage, 'file size', fatal=False)) + filesize = int_or_none(self._extract_filesize(webpage)) return { 'id': video_id, @@ -41,6 +40,10 @@ class SharedBaseIE(InfoExtractor): return compat_b64decode(self._html_search_meta( 'full:title', webpage, 'title')).decode('utf-8') + def _extract_filesize(self, webpage): + return self._html_search_meta( + 'full:size', webpage, 'file size', fatal=False) + class SharedIE(SharedBaseIE): IE_DESC = 'shared.sx' @@ -88,19 +91,27 @@ class VivoIE(SharedBaseIE): 'id': 'd7ddda0e78', 'ext': 'mp4', 'title': 'Chicken', - 'filesize': 528031, + 'filesize': 515659, }, } def _extract_title(self, webpage): - data_title = self._search_regex( + title = self._html_search_regex( r'data-name\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1', webpage, 'title', default=None, group='title') - if data_title: - return unescapeHTML(re.sub(r"\.[a-z0-9]{3,4}$", "", data_title)) + if title: + ext = determine_ext(title) + if ext.lower() in KNOWN_EXTENSIONS: + title = title.rpartition('.' + ext)[0] + return title return self._og_search_title(webpage) - def _extract_video_url(self, webpage, video_id, *args): + def _extract_filesize(self, webpage): + return parse_filesize(self._search_regex( + r'data-type=["\']video["\'][^>]*>Watch.*?<strong>\s*\((.+?)\)', + webpage, 'filesize', fatal=False)) + + def _extract_video_url(self, webpage, video_id, url): def decode_url(encoded_url): return compat_b64decode(encoded_url).decode('utf-8') From ea7538209468f630075d08d44ef7b0119f78d2eb Mon Sep 17 00:00:00 2001 From: smed79 <1873139+smed79@users.noreply.github.com> Date: Wed, 22 May 2019 21:30:17 +0100 Subject: [PATCH 08/46] [openload] Add support for oload.press (#21135) --- youtube_dl/extractor/openload.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index a8e906858..b96be6f64 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -244,7 +244,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space|services)|oladblock\.(?:services|xyz|me)|openloed\.co)' + _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|press|pw|live|space|services)|oladblock\.(?:services|xyz|me)|openloed\.co)' _VALID_URL = r'''(?x) https?:// (?P<host> @@ -357,6 +357,9 @@ class OpenloadIE(InfoExtractor): }, { 'url': 'https://oload.services/embed/bs1NWj1dCag/', 'only_matching': True, + }, { + 'url': 'https://oload.press/embed/drTBl1aOTvk/', + 'only_matching': True, }, { 'url': 'https://oladblock.services/f/b8NWEgkqNLI/', 'only_matching': True, From 612300a686fd83d475b7fddc17cb2ccd8ca0b5ef Mon Sep 17 00:00:00 2001 From: ealgase <mostdigitsofpi@gmail.com> Date: Wed, 22 May 2019 16:38:48 -0400 Subject: [PATCH 09/46] [novamov] Remove extractors (#21077) Sites no longer exist --- youtube_dl/extractor/extractors.py | 7 - youtube_dl/extractor/generic.py | 13 -- youtube_dl/extractor/novamov.py | 212 ----------------------------- 3 files changed, 232 deletions(-) delete mode 100644 youtube_dl/extractor/novamov.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3037b5a45..e5aee96c2 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -772,13 +772,6 @@ from .nova import ( NovaEmbedIE, NovaIE, ) -from .novamov import ( - AuroraVidIE, - CloudTimeIE, - NowVideoIE, - VideoWeedIE, - WholeCloudIE, -) from .nowness import ( NownessIE, NownessPlaylistIE, diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 3a13c62eb..eeb0d25f6 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -2583,19 +2583,6 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group(1), 'Mpora') - # Look for embedded NovaMov-based player - mobj = re.search( - r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\']) - (?P<url>http://(?:(?:embed|www)\.)? - (?:novamov\.com| - nowvideo\.(?:ch|sx|eu|at|ag|co)| - videoweed\.(?:es|com)| - movshare\.(?:net|sx|ag)| - divxstage\.(?:eu|net|ch|co|at|ag)) - /embed\.php.+?)\1''', webpage) - if mobj is not None: - return self.url_result(mobj.group('url')) - # Look for embedded Facebook player facebook_urls = FacebookIE._extract_urls(webpage) if facebook_urls: diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py deleted file mode 100644 index 829c71960..000000000 --- a/youtube_dl/extractor/novamov.py +++ /dev/null @@ -1,212 +0,0 @@ -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..compat import compat_urlparse -from ..utils import ( - ExtractorError, - NO_DEFAULT, - sanitized_Request, - urlencode_postdata, -) - - -class NovaMovIE(InfoExtractor): - IE_NAME = 'novamov' - IE_DESC = 'NovaMov' - - _VALID_URL_TEMPLATE = r'''(?x) - http:// - (?: - (?:www\.)?%(host)s/(?:file|video|mobile/\#/videos)/| - (?:(?:embed|www)\.)%(host)s/embed(?:\.php|/)?\?(?:.*?&)?\bv= - ) - (?P<id>[a-z\d]{13}) - ''' - _VALID_URL = _VALID_URL_TEMPLATE % {'host': r'novamov\.com'} - - _HOST = 'www.novamov.com' - - _FILE_DELETED_REGEX = r'This file no longer exists on our servers!</h2>' - _FILEKEY_REGEX = r'flashvars\.filekey=(?P<filekey>"?[^"]+"?);' - _TITLE_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>' - _DESCRIPTION_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>' - _URL_TEMPLATE = 'http://%s/video/%s' - - _TEST = None - - def _check_existence(self, webpage, video_id): - if re.search(self._FILE_DELETED_REGEX, webpage) is not None: - raise ExtractorError('Video %s does not exist' % video_id, expected=True) - - def _real_extract(self, url): - video_id = self._match_id(url) - - url = self._URL_TEMPLATE % (self._HOST, video_id) - - webpage = self._download_webpage( - url, video_id, 'Downloading video page') - - self._check_existence(webpage, video_id) - - def extract_filekey(default=NO_DEFAULT): - filekey = self._search_regex( - self._FILEKEY_REGEX, webpage, 'filekey', default=default) - if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'): - return self._search_regex( - r'var\s+%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default) - else: - return filekey - - filekey = extract_filekey(default=None) - - if not filekey: - fields = self._hidden_inputs(webpage) - post_url = self._search_regex( - r'<form[^>]+action=(["\'])(?P<url>.+?)\1', webpage, - 'post url', default=url, group='url') - if not post_url.startswith('http'): - post_url = compat_urlparse.urljoin(url, post_url) - request = sanitized_Request( - post_url, urlencode_postdata(fields)) - request.add_header('Content-Type', 'application/x-www-form-urlencoded') - request.add_header('Referer', post_url) - webpage = self._download_webpage( - request, video_id, 'Downloading continue to the video page') - self._check_existence(webpage, video_id) - - filekey = extract_filekey() - - title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title') - description = self._html_search_regex(self._DESCRIPTION_REGEX, webpage, 'description', default='', fatal=False) - - api_response = self._download_webpage( - 'http://%s/api/player.api.php?key=%s&file=%s' % (self._HOST, filekey, video_id), video_id, - 'Downloading video api response') - - response = compat_urlparse.parse_qs(api_response) - - if 'error_msg' in response: - raise ExtractorError('%s returned error: %s' % (self.IE_NAME, response['error_msg'][0]), expected=True) - - video_url = response['url'][0] - - return { - 'id': video_id, - 'url': video_url, - 'title': title, - 'description': description - } - - -class WholeCloudIE(NovaMovIE): - IE_NAME = 'wholecloud' - IE_DESC = 'WholeCloud' - - _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': r'(?:wholecloud\.net|movshare\.(?:net|sx|ag))'} - - _HOST = 'www.wholecloud.net' - - _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<' - _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>' - _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>' - - _TEST = { - 'url': 'http://www.wholecloud.net/video/559e28be54d96', - 'md5': 'abd31a2132947262c50429e1d16c1bfd', - 'info_dict': { - 'id': '559e28be54d96', - 'ext': 'flv', - 'title': 'dissapeared image', - 'description': 'optical illusion dissapeared image magic illusion', - } - } - - -class NowVideoIE(NovaMovIE): - IE_NAME = 'nowvideo' - IE_DESC = 'NowVideo' - - _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': r'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'} - - _HOST = 'www.nowvideo.to' - - _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<' - _TITLE_REGEX = r'<h4>([^<]+)</h4>' - _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>' - - _TEST = { - 'url': 'http://www.nowvideo.sx/video/f1d6fce9a968b', - 'md5': '12c82cad4f2084881d8bc60ee29df092', - 'info_dict': { - 'id': 'f1d6fce9a968b', - 'ext': 'flv', - 'title': 'youtubedl test video BaWjenozKc', - 'description': 'Description', - }, - } - - -class VideoWeedIE(NovaMovIE): - IE_NAME = 'videoweed' - IE_DESC = 'VideoWeed' - - _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': r'videoweed\.(?:es|com)'} - - _HOST = 'www.videoweed.es' - - _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<' - _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>' - _URL_TEMPLATE = 'http://%s/file/%s' - - _TEST = { - 'url': 'http://www.videoweed.es/file/b42178afbea14', - 'md5': 'abd31a2132947262c50429e1d16c1bfd', - 'info_dict': { - 'id': 'b42178afbea14', - 'ext': 'flv', - 'title': 'optical illusion dissapeared image magic illusion', - 'description': '' - }, - } - - -class CloudTimeIE(NovaMovIE): - IE_NAME = 'cloudtime' - IE_DESC = 'CloudTime' - - _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': r'cloudtime\.to'} - - _HOST = 'www.cloudtime.to' - - _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<' - _TITLE_REGEX = r'<div[^>]+class=["\']video_det["\'][^>]*>\s*<strong>([^<]+)</strong>' - - _TEST = None - - -class AuroraVidIE(NovaMovIE): - IE_NAME = 'auroravid' - IE_DESC = 'AuroraVid' - - _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': r'auroravid\.to'} - - _HOST = 'www.auroravid.to' - - _FILE_DELETED_REGEX = r'This file no longer exists on our servers!<' - - _TESTS = [{ - 'url': 'http://www.auroravid.to/video/4rurhn9x446jj', - 'md5': '7205f346a52bbeba427603ba10d4b935', - 'info_dict': { - 'id': '4rurhn9x446jj', - 'ext': 'flv', - 'title': 'search engine optimization', - 'description': 'search engine optimization is used to rank the web page in the google search engine' - }, - 'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)' - }, { - 'url': 'http://www.auroravid.to/embed/?v=4rurhn9x446jj', - 'only_matching': True, - }] From 186d185b6ecdee102866777121d6abe9ed7f59ba Mon Sep 17 00:00:00 2001 From: Malte Kiefer <malte.kiefer@mailgermania.de> Date: Wed, 22 May 2019 22:46:20 +0200 Subject: [PATCH 10/46] [streamcloud] Reduce waiting time to 6 seconds (#21092) --- youtube_dl/extractor/streamcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py index 4a410611d..b97bb4374 100644 --- a/youtube_dl/extractor/streamcloud.py +++ b/youtube_dl/extractor/streamcloud.py @@ -45,7 +45,7 @@ class StreamcloudIE(InfoExtractor): value="([^"]*)" ''', orig_webpage) - self._sleep(12, video_id) + self._sleep(6, video_id) webpage = self._download_webpage( url, video_id, data=urlencode_postdata(fields), headers={ From bbf1defe586f4b4cb7b35aa3da67c5dc786d9a2c Mon Sep 17 00:00:00 2001 From: Georgi Saev <georgi.saev@gmail.com> Date: Wed, 22 May 2019 23:51:50 +0300 Subject: [PATCH 11/46] [bitchute] Fix uploader extraction (#21076) --- youtube_dl/extractor/bitchute.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py index 4f39424f5..1d69dafbd 100644 --- a/youtube_dl/extractor/bitchute.py +++ b/youtube_dl/extractor/bitchute.py @@ -65,8 +65,9 @@ class BitChuteIE(InfoExtractor): webpage, default=None) or self._html_search_meta( 'twitter:image:src', webpage, 'thumbnail') uploader = self._html_search_regex( - r'(?s)<p\b[^>]+\bclass=["\']video-author[^>]+>(.+?)</p>', webpage, - 'uploader', fatal=False) + (r'(?s)<div class=["\']channel-banner.*?<p\b[^>]+\bclass=["\']name[^>]+>(.+?)</p>', + r'(?s)<p\b[^>]+\bclass=["\']video-author[^>]+>(.+?)</p>'), + webpage, 'uploader', fatal=False) return { 'id': video_id, From 2c53c0ebc63b7fbb36d05491d5d3796d3e511e26 Mon Sep 17 00:00:00 2001 From: NRTICN <50528161+NRTICN@users.noreply.github.com> Date: Wed, 22 May 2019 20:56:54 +0000 Subject: [PATCH 12/46] [pornhub] Use https (#21061) --- youtube_dl/extractor/pornhub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index bf8f0be88..cb59d526f 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -170,7 +170,7 @@ class PornHubIE(PornHubBaseIE): def dl_webpage(platform): self._set_cookie(host, 'platform', platform) return self._download_webpage( - 'http://www.%s/view_video.php?viewkey=%s' % (host, video_id), + 'https://www.%s/view_video.php?viewkey=%s' % (host, video_id), video_id, 'Downloading %s webpage' % platform) webpage = dl_webpage('pc') From afd4985f72a6641907aee1cd0b4b42da524b0ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 23 May 2019 06:06:49 +0700 Subject: [PATCH 13/46] [travis] Force dist to Ubuntu Trusty by default According to https://blog.travis-ci.com/2019-04-15-xenial-default-build-environment Ubuntu Xenial is now default, but it lacks python 2.6, 3.2 and 3.3 support needed by tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 82e81d078..6d16c2955 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "3.6" - "pypy" - "pypy3" +dist: trusty env: - YTDL_TEST_SET=core - YTDL_TEST_SET=download From 9c5f2988b91609d49f7010ac580376f42e01d4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 23 May 2019 23:38:01 +0700 Subject: [PATCH 14/46] [criterion] Remove extractor (closes #21195) --- youtube_dl/extractor/criterion.py | 39 ------------------------------ youtube_dl/extractor/extractors.py | 1 - 2 files changed, 40 deletions(-) delete mode 100644 youtube_dl/extractor/criterion.py diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py deleted file mode 100644 index f7815b905..000000000 --- a/youtube_dl/extractor/criterion.py +++ /dev/null @@ -1,39 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor - - -class CriterionIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?criterion\.com/films/(?P<id>[0-9]+)-.+' - _TEST = { - 'url': 'http://www.criterion.com/films/184-le-samourai', - 'md5': 'bc51beba55685509883a9a7830919ec3', - 'info_dict': { - 'id': '184', - 'ext': 'mp4', - 'title': 'Le Samouraï', - 'description': 'md5:a2b4b116326558149bef81f76dcbb93f', - 'thumbnail': r're:^https?://.*\.jpg$', - } - } - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - final_url = self._search_regex( - r'so\.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url') - title = self._og_search_title(webpage) - description = self._html_search_meta('description', webpage) - thumbnail = self._search_regex( - r'so\.addVariable\("thumbnailURL", "(.+?)"\)\;', - webpage, 'thumbnail url') - - return { - 'id': video_id, - 'url': final_url, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - } diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index e5aee96c2..7705f9bdd 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -240,7 +240,6 @@ from .condenast import CondeNastIE from .corus import CorusIE from .cracked import CrackedIE from .crackle import CrackleIE -from .criterion import CriterionIE from .crooksandliars import CrooksAndLiarsIE from .crunchyroll import ( CrunchyrollIE, From 8af49fc276b2cf2154b9342de4b4cd66f9d17af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 23 May 2019 23:48:06 +0700 Subject: [PATCH 15/46] [pornflip] Remove extractor --- youtube_dl/extractor/extractors.py | 1 - youtube_dl/extractor/pornflip.py | 101 ----------------------------- 2 files changed, 102 deletions(-) delete mode 100644 youtube_dl/extractor/pornflip.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 7705f9bdd..eb5efd1e8 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -888,7 +888,6 @@ from .polskieradio import ( from .popcorntv import PopcornTVIE from .porn91 import Porn91IE from .porncom import PornComIE -from .pornflip import PornFlipIE from .pornhd import PornHdIE from .pornhub import ( PornHubIE, diff --git a/youtube_dl/extractor/pornflip.py b/youtube_dl/extractor/pornflip.py deleted file mode 100644 index 025985fbc..000000000 --- a/youtube_dl/extractor/pornflip.py +++ /dev/null @@ -1,101 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..compat import ( - compat_parse_qs, - compat_str, -) -from ..utils import ( - int_or_none, - try_get, - unified_timestamp, -) - - -class PornFlipIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[^/?#&]+)' - _TESTS = [{ - 'url': 'https://www.pornflip.com/v/wz7DfNhMmep', - 'md5': '98c46639849145ae1fd77af532a9278c', - 'info_dict': { - 'id': 'wz7DfNhMmep', - 'ext': 'mp4', - 'title': '2 Amateurs swallow make his dream cumshots true', - 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 112, - 'timestamp': 1481655502, - 'upload_date': '20161213', - 'uploader_id': '106786', - 'uploader': 'figifoto', - 'view_count': int, - 'age_limit': 18, - } - }, { - 'url': 'https://www.pornflip.com/embed/wz7DfNhMmep', - 'only_matching': True, - }, { - 'url': 'https://www.pornflip.com/v/EkRD6-vS2-s', - 'only_matching': True, - }, { - 'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s', - 'only_matching': True, - }, { - 'url': 'https://www.pornflip.com/v/NG9q6Pb_iK8', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage( - 'https://www.pornflip.com/v/%s' % video_id, video_id) - - flashvars = compat_parse_qs(self._search_regex( - r'<embed[^>]+flashvars=(["\'])(?P<flashvars>(?:(?!\1).)+)\1', - webpage, 'flashvars', group='flashvars')) - - title = flashvars['video_vars[title]'][0] - - def flashvar(kind): - return try_get( - flashvars, lambda x: x['video_vars[%s]' % kind][0], compat_str) - - formats = [] - for key, value in flashvars.items(): - if not (value and isinstance(value, list)): - continue - format_url = value[0] - if key == 'video_vars[hds_manifest]': - formats.extend(self._extract_mpd_formats( - format_url, video_id, mpd_id='dash', fatal=False)) - continue - height = self._search_regex( - r'video_vars\[video_urls\]\[(\d+)', key, 'height', default=None) - if not height: - continue - formats.append({ - 'url': format_url, - 'format_id': 'http-%s' % height, - 'height': int_or_none(height), - }) - self._sort_formats(formats) - - uploader = self._html_search_regex( - (r'<span[^>]+class="name"[^>]*>\s*<a[^>]+>\s*<strong>(?P<uploader>[^<]+)', - r'<meta[^>]+content=(["\'])[^>]*\buploaded by (?P<uploader>.+?)\1'), - webpage, 'uploader', fatal=False, group='uploader') - - return { - 'id': video_id, - 'formats': formats, - 'title': title, - 'thumbnail': flashvar('big_thumb'), - 'duration': int_or_none(flashvar('duration')), - 'timestamp': unified_timestamp(self._html_search_meta( - 'uploadDate', webpage, 'timestamp')), - 'uploader_id': flashvar('author_id'), - 'uploader': uploader, - 'view_count': int_or_none(flashvar('views')), - 'age_limit': 18, - } From f856816b940d95e995c2b9995d01097ab144a1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 23 May 2019 23:52:11 +0700 Subject: [PATCH 16/46] [extractor/common] Strip src attribute for HTML5 entries code (closes #18485, closes #21169) --- youtube_dl/extractor/common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 937237b3f..9c3e9eec6 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -67,6 +67,7 @@ from ..utils import ( sanitized_Request, sanitize_filename, str_or_none, + strip_or_none, unescapeHTML, unified_strdate, unified_timestamp, @@ -2480,7 +2481,7 @@ class InfoExtractor(object): 'subtitles': {}, } media_attributes = extract_attributes(media_tag) - src = media_attributes.get('src') + src = strip_or_none(media_attributes.get('src')) if src: _, formats = _media_formats(src, media_type) media_info['formats'].extend(formats) @@ -2490,7 +2491,7 @@ class InfoExtractor(object): s_attr = extract_attributes(source_tag) # data-video-src and data-src are non standard but seen # several times in the wild - src = dict_get(s_attr, ('src', 'data-video-src', 'data-src')) + src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src'))) if not src: continue f = parse_content_type(s_attr.get('type')) @@ -2533,7 +2534,7 @@ class InfoExtractor(object): track_attributes = extract_attributes(track_tag) kind = track_attributes.get('kind') if not kind or kind in ('subtitles', 'captions'): - src = track_attributes.get('src') + src = strip_or_none(track_attributes.get('src')) if not src: continue lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label') From 53cd37bac50e7a927deba5b67a4412301b96230d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Thu, 23 May 2019 23:58:35 +0700 Subject: [PATCH 17/46] [utils] Improve strip_or_none --- test/test_utils.py | 13 +++++++++++++ youtube_dl/utils.py | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 9ef0e422b..71980b3fc 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -73,6 +73,7 @@ from youtube_dl.utils import ( smuggle_url, str_to_int, strip_jsonp, + strip_or_none, timeconvert, unescapeHTML, unified_strdate, @@ -752,6 +753,18 @@ class TestUtil(unittest.TestCase): d = json.loads(stripped) self.assertEqual(d, {'status': 'success'}) + def test_strip_or_none(self): + self.assertEqual(strip_or_none(' abc'), 'abc') + self.assertEqual(strip_or_none('abc '), 'abc') + self.assertEqual(strip_or_none(' abc '), 'abc') + self.assertEqual(strip_or_none('\tabc\t'), 'abc') + self.assertEqual(strip_or_none('\n\tabc\n\t'), 'abc') + self.assertEqual(strip_or_none('abc'), 'abc') + self.assertEqual(strip_or_none(''), '') + self.assertEqual(strip_or_none(None), None) + self.assertEqual(strip_or_none(42), None) + self.assertEqual(strip_or_none([]), None) + def test_uppercase_escape(self): self.assertEqual(uppercase_escape('aä'), 'aä') self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 9be9b2e76..ead9bd862 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1951,8 +1951,8 @@ def bool_or_none(v, default=None): return v if isinstance(v, bool) else default -def strip_or_none(v): - return None if v is None else v.strip() +def strip_or_none(v, default=None): + return v.strip() if isinstance(v, compat_str) else default def url_or_none(url): From 11ec06de7f01bced1f4f4b6484e5190cbb9ed9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 24 May 2019 00:35:46 +0700 Subject: [PATCH 18/46] [24video] Add support for 24video.site (closes #21193) --- youtube_dl/extractor/twentyfourvideo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py index 4b3b3e705..d16f55500 100644 --- a/youtube_dl/extractor/twentyfourvideo.py +++ b/youtube_dl/extractor/twentyfourvideo.py @@ -14,7 +14,7 @@ from ..utils import ( class TwentyFourVideoIE(InfoExtractor): IE_NAME = '24video' - _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)' + _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult|site))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)' _TESTS = [{ 'url': 'http://www.24video.net/video/view/1044982', @@ -42,6 +42,9 @@ class TwentyFourVideoIE(InfoExtractor): }, { 'url': 'http://www.24video.tube/video/view/2363750', 'only_matching': True, + }, { + 'url': 'https://www.24video.site/video/view/2640421', + 'only_matching': True, }] def _real_extract(self, url): From f4cc2ca503239aadd5cb75f83cb873bc5816dfdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 24 May 2019 00:38:06 +0700 Subject: [PATCH 19/46] [24video] Add support for porno.24video.net (closes #21194) --- youtube_dl/extractor/twentyfourvideo.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py index d16f55500..1d66eeaff 100644 --- a/youtube_dl/extractor/twentyfourvideo.py +++ b/youtube_dl/extractor/twentyfourvideo.py @@ -14,7 +14,18 @@ from ..utils import ( class TwentyFourVideoIE(InfoExtractor): IE_NAME = '24video' - _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sexy?|tube|adult|site))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)' + _VALID_URL = r'''(?x) + https?:// + (?P<host> + (?:(?:www|porno)\.)?24video\. + (?:net|me|xxx|sexy?|tube|adult|site) + )/ + (?: + video/(?:(?:view|xml)/)?| + player/new24_play\.swf\?id= + ) + (?P<id>\d+) + ''' _TESTS = [{ 'url': 'http://www.24video.net/video/view/1044982', @@ -45,6 +56,9 @@ class TwentyFourVideoIE(InfoExtractor): }, { 'url': 'https://www.24video.site/video/view/2640421', 'only_matching': True, + }, { + 'url': 'https://porno.24video.net/video/2640421-vsya-takaya-gibkaya-i-v-masle', + 'only_matching': True, }] def _real_extract(self, url): From 3fe774722b0855d572b6885ff20fb6f4f96f3e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20P=C3=B6schel?= <github@basicmaster.de> Date: Sun, 19 May 2019 18:11:17 +0200 Subject: [PATCH 20/46] [srgssrplay] Add support for popupvideoplayer URLs --- youtube_dl/extractor/srgssr.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py index bb73eb1d5..ff1b67876 100644 --- a/youtube_dl/extractor/srgssr.py +++ b/youtube_dl/extractor/srgssr.py @@ -106,7 +106,7 @@ class SRGSSRIE(InfoExtractor): class SRGSSRPlayIE(InfoExtractor): IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites' - _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)' + _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/(?:[^/]+/|popup)(?P<type>video|audio)(?:/[^?]+|player)\?id=(?P<id>[0-9a-f\-]{36}|\d+)' _TESTS = [{ 'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5', @@ -163,6 +163,17 @@ class SRGSSRPlayIE(InfoExtractor): # m3u8 download 'skip_download': True, } + }, { + 'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01', + 'md5': 'f6247aa7c905b81c9ba7f50fb22e2fbd', + 'info_dict': { + 'id': 'c4dba0ca-e75b-43b2-a34f-f708a4932e01', + 'ext': 'mp4', + 'upload_date': '20190122', + 'title': 'Erster Selfie-Stick (1983)', + 'description': 'md5:23a6b40024e583137e4137f5946543c1', + 'timestamp': 1548155133, + } }] def _real_extract(self, url): From 25b83c2a0e29c75372f0ce26d2b4ecf493e8b28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 24 May 2019 00:43:22 +0700 Subject: [PATCH 21/46] [srgssrplay] Improve _VALID_URL (closes #21155) --- youtube_dl/extractor/srgssr.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py index ff1b67876..170dce87f 100644 --- a/youtube_dl/extractor/srgssr.py +++ b/youtube_dl/extractor/srgssr.py @@ -106,7 +106,16 @@ class SRGSSRIE(InfoExtractor): class SRGSSRPlayIE(InfoExtractor): IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites' - _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/(?:[^/]+/|popup)(?P<type>video|audio)(?:/[^?]+|player)\?id=(?P<id>[0-9a-f\-]{36}|\d+)' + _VALID_URL = r'''(?x) + https?:// + (?:(?:www|play)\.)? + (?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/ + (?: + [^/]+/(?P<type>video|audio)/[^?]+| + popup(?P<type_2>video|audio)player + ) + \?id=(?P<id>[0-9a-f\-]{36}|\d+) + ''' _TESTS = [{ 'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5', @@ -165,18 +174,13 @@ class SRGSSRPlayIE(InfoExtractor): } }, { 'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01', - 'md5': 'f6247aa7c905b81c9ba7f50fb22e2fbd', - 'info_dict': { - 'id': 'c4dba0ca-e75b-43b2-a34f-f708a4932e01', - 'ext': 'mp4', - 'upload_date': '20190122', - 'title': 'Erster Selfie-Stick (1983)', - 'description': 'md5:23a6b40024e583137e4137f5946543c1', - 'timestamp': 1548155133, - } + 'only_matching': True, }] def _real_extract(self, url): - bu, media_type, media_id = re.match(self._VALID_URL, url).groups() + mobj = re.match(self._VALID_URL, url) + bu = mobj.group('bu') + media_type = mobj.group('type') or mobj.group('type_2') + media_id = mobj.group('id') # other info can be extracted from url + '&layout=json' return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR') From 0d297518904f8537e8103052d19f0940a272413d Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Sat, 25 May 2019 23:14:47 +0100 Subject: [PATCH 22/46] [youtube] improve DRM protected videos detection(#1774) --- youtube_dl/extractor/youtube.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 06005f8d2..5f1957a59 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1789,9 +1789,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError( 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id) - if video_info.get('license_info'): - raise ExtractorError('This video is DRM protected.', expected=True) - video_details = try_get( player_response, lambda x: x['videoDetails'], dict) or {} @@ -1927,7 +1924,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): formats = [] for url_data_str in encoded_url_map.split(','): url_data = compat_parse_qs(url_data_str) - if 'itag' not in url_data or 'url' not in url_data: + if 'itag' not in url_data or 'url' not in url_data or url_data.get('drm_families'): continue stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0])) # Unsupported FORMAT_STREAM_TYPE_OTF @@ -2323,6 +2320,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '"token" parameter not in video info for unknown reason', video_id=video_id) + if not formats and (video_info.get('license_info') or try_get(player_response, lambda x: x['streamingData']['licenseInfos'])): + raise ExtractorError('This video is DRM protected.', expected=True) + self._sort_formats(formats) self.mark_watched(video_id, video_info, player_response) From ead467a9c1cab3f26279ba9c57cf18598507ba79 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Tue, 28 May 2019 04:58:12 +0100 Subject: [PATCH 23/46] [rtp] fix extraction(closes #15099) --- youtube_dl/extractor/rtp.py | 83 ++++++++++++++----------------------- 1 file changed, 30 insertions(+), 53 deletions(-) diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py index 533ee27cb..02986f442 100644 --- a/youtube_dl/extractor/rtp.py +++ b/youtube_dl/extractor/rtp.py @@ -1,9 +1,11 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor +from ..utils import ( + determine_ext, + js_to_json, +) class RTPIE(InfoExtractor): @@ -18,10 +20,6 @@ class RTPIE(InfoExtractor): 'description': 'As paixões musicais de António Cartaxo e António Macedo', 'thumbnail': r're:^https?://.*\.jpg', }, - 'params': { - # rtmp download - 'skip_download': True, - }, }, { 'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas', 'only_matching': True, @@ -33,57 +31,36 @@ class RTPIE(InfoExtractor): webpage = self._download_webpage(url, video_id) title = self._html_search_meta( 'twitter:title', webpage, display_name='title', fatal=True) - description = self._html_search_meta('description', webpage) - thumbnail = self._og_search_thumbnail(webpage) - player_config = self._search_regex( - r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config') - config = self._parse_json(player_config, video_id) - - path, ext = config.get('file').rsplit('.', 1) - formats = [{ - 'format_id': 'rtmp', - 'ext': ext, - 'vcodec': config.get('type') == 'audio' and 'none' or None, - 'preference': -2, - 'url': 'rtmp://{streamer:s}/{application:s}'.format(**config), - 'app': config.get('application'), - 'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path), - 'page_url': url, - 'rtmp_live': config.get('live', False), - 'player_url': 'http://programas.rtp.pt/play/player.swf?v3', - 'rtmp_real_time': True, - }] - - # Construct regular HTTP download URLs - replacements = { - 'audio': { - 'format_id': 'mp3', - 'pattern': r'^nas2\.share/wavrss/', - 'repl': 'http://rsspod.rtp.pt/podcasts/', - 'vcodec': 'none', - }, - 'video': { - 'format_id': 'mp4_h264', - 'pattern': r'^nas2\.share/h264/', - 'repl': 'http://rsspod.rtp.pt/videocasts/', - 'vcodec': 'h264', - }, - } - r = replacements[config['type']] - if re.match(r['pattern'], config['file']) is not None: - formats.append({ - 'format_id': r['format_id'], - 'url': re.sub(r['pattern'], r['repl'], config['file']), - 'vcodec': r['vcodec'], - }) - - self._sort_formats(formats) + config = self._parse_json(self._search_regex( + r'(?s)RTPPlayer\(({.+?})\);', webpage, + 'player config'), video_id, js_to_json) + file_url = config['file'] + ext = determine_ext(file_url) + if ext == 'm3u8': + file_key = config.get('fileKey') + formats = self._extract_m3u8_formats( + file_url, video_id, 'mp4', 'm3u8_native', + m3u8_id='hls', fatal=file_key) + if file_key: + formats.append({ + 'url': 'https://cdn-ondemand.rtp.pt' + file_key, + 'preference': 1, + }) + self._sort_formats(formats) + else: + formats = [{ + 'url': file_url, + 'ext': ext, + }] + if config.get('mediaType') == 'audio': + for f in formats: + f['vcodec'] = 'none' return { 'id': video_id, 'title': title, 'formats': formats, - 'description': description, - 'thumbnail': thumbnail, + 'description': self._html_search_meta(['description', 'twitter:description'], webpage), + 'thumbnail': config.get('poster') or self._og_search_thumbnail(webpage), } From 33b2218b2f67b4c06d45e28035968320e0a0bf05 Mon Sep 17 00:00:00 2001 From: bitraid <bitraid@protonmail.ch> Date: Tue, 28 May 2019 15:31:11 +0300 Subject: [PATCH 24/46] [LiveLeak] Check if the original videos exist (closes #21206) (#21208) --- youtube_dl/extractor/liveleak.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py index 5df14bb41..4ac437c8b 100644 --- a/youtube_dl/extractor/liveleak.py +++ b/youtube_dl/extractor/liveleak.py @@ -82,6 +82,10 @@ class LiveLeakIE(InfoExtractor): }, { 'url': 'https://www.liveleak.com/view?t=HvHi_1523016227', 'only_matching': True, + }, { + # No original video + 'url': 'https://www.liveleak.com/view?t=C26ZZ_1558612804', + 'only_matching': True, }] @staticmethod @@ -134,11 +138,13 @@ class LiveLeakIE(InfoExtractor): orig_url = re.sub(r'\.mp4\.[^.]+', '', a_format['url']) if a_format['url'] != orig_url: format_id = a_format.get('format_id') - formats.append({ - 'format_id': 'original' + ('-' + format_id if format_id else ''), - 'url': orig_url, - 'preference': 1, - }) + format_id = 'original' + ('-' + format_id if format_id else '') + if self._is_valid_url(orig_url, video_id, format_id): + formats.append({ + 'format_id': format_id, + 'url': orig_url, + 'preference': 1, + }) self._sort_formats(formats) info_dict['formats'] = formats From 26a87972a9dcd4eac6112c06372745d6d0983f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Wed, 29 May 2019 04:42:19 +0700 Subject: [PATCH 25/46] [viki] Switch to HTTPS (closes #21001) --- youtube_dl/extractor/viki.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index 546de95d8..b0dcdc0e6 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -21,7 +21,7 @@ from ..utils import ( class VikiBaseIE(InfoExtractor): _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/' _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com' - _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s' + _API_URL_TEMPLATE = 'https://api.viki.io%s&sig=%s' _APP = '100005a' _APP_VERSION = '2.2.5.1428709186' @@ -377,7 +377,7 @@ class VikiChannelIE(VikiBaseIE): for video in page['response']: video_id = video['id'] entries.append(self.url_result( - 'http://www.viki.com/videos/%s' % video_id, 'Viki')) + 'https://www.viki.com/videos/%s' % video_id, 'Viki')) if not page['pagination']['next']: break From 0e2dd3fcbce3a01dc4f6ce3911f5f279c3827edf Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Sat, 1 Jun 2019 11:16:44 +0100 Subject: [PATCH 26/46] [vrv] extract adaptive_hls formats(closes #21243) --- youtube_dl/extractor/vrv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py index b698bf66c..c814a8a4a 100644 --- a/youtube_dl/extractor/vrv.py +++ b/youtube_dl/extractor/vrv.py @@ -130,7 +130,7 @@ class VRVIE(VRVBaseIE): self._TOKEN_SECRET = token_credentials['oauth_token_secret'] def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang): - if not url or stream_format not in ('hls', 'dash'): + if not url or stream_format not in ('hls', 'dash', 'adaptive_hls'): return [] stream_id_list = [] if audio_lang: @@ -140,7 +140,7 @@ class VRVIE(VRVBaseIE): format_id = stream_format if stream_id_list: format_id += '-' + '-'.join(stream_id_list) - if stream_format == 'hls': + if 'hls' in stream_format: adaptive_formats = self._extract_m3u8_formats( url, video_id, 'mp4', m3u8_id=format_id, note='Downloading %s information' % format_id, From c5eb75b35a709baa134fb96e774a4b712cc65e93 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Sat, 1 Jun 2019 15:12:30 +0100 Subject: [PATCH 27/46] [prosiebensat1] add support for new API(closes #21272) --- youtube_dl/extractor/prosiebensat1.py | 197 ++++++++++++++++---------- 1 file changed, 121 insertions(+), 76 deletions(-) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index 7d11c2b9b..e19a470a5 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -16,6 +16,11 @@ from ..utils import ( class ProSiebenSat1BaseIE(InfoExtractor): + _GEO_COUNTRIES = ['DE'] + _ACCESS_ID = None + _SUPPORTED_PROTOCOLS = 'dash:clear,hls:clear,progressive:clear' + _V4_BASE_URL = 'https://vas-v4.p7s1video.net/4.0/get' + def _extract_video_info(self, url, clip_id): client_location = url @@ -31,93 +36,128 @@ class ProSiebenSat1BaseIE(InfoExtractor): if video.get('is_protected') is True: raise ExtractorError('This video is DRM protected.', expected=True) - duration = float_or_none(video.get('duration')) - source_ids = [compat_str(source['id']) for source in video['sources']] - - client_id = self._SALT[:2] + sha1(''.join([clip_id, self._SALT, self._TOKEN, client_location, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest() - - sources = self._download_json( - 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id, - clip_id, 'Downloading sources JSON', query={ - 'access_token': self._TOKEN, - 'client_id': client_id, - 'client_location': client_location, - 'client_name': self._CLIENT_NAME, - }) - server_id = sources['server_id'] - - def fix_bitrate(bitrate): - bitrate = int_or_none(bitrate) - if not bitrate: - return None - return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate - formats = [] - for source_id in source_ids: - client_id = self._SALT[:2] + sha1(''.join([self._SALT, clip_id, self._TOKEN, server_id, client_location, source_id, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest() - urls = self._download_json( - 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id, - clip_id, 'Downloading urls JSON', fatal=False, query={ + if self._ACCESS_ID: + raw_ct = self._ENCRYPTION_KEY + clip_id + self._IV + self._ACCESS_ID + server_token = (self._download_json( + self._V4_BASE_URL + 'protocols', clip_id, + 'Downloading protocols JSON', + headers=self.geo_verification_headers(), query={ + 'access_id': self._ACCESS_ID, + 'client_token': sha1((raw_ct).encode()).hexdigest(), + 'video_id': clip_id, + }, fatal=False) or {}).get('server_token') + if server_token: + urls = (self._download_json( + self._V4_BASE_URL + 'urls', clip_id, 'Downloading urls JSON', query={ + 'access_id': self._ACCESS_ID, + 'client_token': sha1((raw_ct + server_token + self._SUPPORTED_PROTOCOLS).encode()).hexdigest(), + 'protocols': self._SUPPORTED_PROTOCOLS, + 'server_token': server_token, + 'video_id': clip_id, + }, fatal=False) or {}).get('urls') or {} + for protocol, variant in urls.items(): + source_url = variant.get('clear', {}).get('url') + if not source_url: + continue + if protocol == 'dash': + formats.extend(self._extract_mpd_formats( + source_url, clip_id, mpd_id=protocol, fatal=False)) + elif protocol == 'hls': + formats.extend(self._extract_m3u8_formats( + source_url, clip_id, 'mp4', 'm3u8_native', + m3u8_id=protocol, fatal=False)) + else: + formats.append({ + 'url': source_url, + 'format_id': protocol, + }) + if not formats: + source_ids = [compat_str(source['id']) for source in video['sources']] + + client_id = self._SALT[:2] + sha1(''.join([clip_id, self._SALT, self._TOKEN, client_location, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest() + + sources = self._download_json( + 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id, + clip_id, 'Downloading sources JSON', query={ 'access_token': self._TOKEN, 'client_id': client_id, 'client_location': client_location, 'client_name': self._CLIENT_NAME, - 'server_id': server_id, - 'source_ids': source_id, }) - if not urls: - continue - if urls.get('status_code') != 0: - raise ExtractorError('This video is unavailable', expected=True) - urls_sources = urls['sources'] - if isinstance(urls_sources, dict): - urls_sources = urls_sources.values() - for source in urls_sources: - source_url = source.get('url') - if not source_url: + server_id = sources['server_id'] + + def fix_bitrate(bitrate): + bitrate = int_or_none(bitrate) + if not bitrate: + return None + return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate + + for source_id in source_ids: + client_id = self._SALT[:2] + sha1(''.join([self._SALT, clip_id, self._TOKEN, server_id, client_location, source_id, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest() + urls = self._download_json( + 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id, + clip_id, 'Downloading urls JSON', fatal=False, query={ + 'access_token': self._TOKEN, + 'client_id': client_id, + 'client_location': client_location, + 'client_name': self._CLIENT_NAME, + 'server_id': server_id, + 'source_ids': source_id, + }) + if not urls: continue - protocol = source.get('protocol') - mimetype = source.get('mimetype') - if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m': - formats.extend(self._extract_f4m_formats( - source_url, clip_id, f4m_id='hds', fatal=False)) - elif mimetype == 'application/x-mpegURL': - formats.extend(self._extract_m3u8_formats( - source_url, clip_id, 'mp4', 'm3u8_native', - m3u8_id='hls', fatal=False)) - elif mimetype == 'application/dash+xml': - formats.extend(self._extract_mpd_formats( - source_url, clip_id, mpd_id='dash', fatal=False)) - else: - tbr = fix_bitrate(source['bitrate']) - if protocol in ('rtmp', 'rtmpe'): - mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url) - if not mobj: - continue - path = mobj.group('path') - mp4colon_index = path.rfind('mp4:') - app = path[:mp4colon_index] - play_path = path[mp4colon_index:] - formats.append({ - 'url': '%s/%s' % (mobj.group('url'), app), - 'app': app, - 'play_path': play_path, - 'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf', - 'page_url': 'http://www.prosieben.de', - 'tbr': tbr, - 'ext': 'flv', - 'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''), - }) + if urls.get('status_code') != 0: + raise ExtractorError('This video is unavailable', expected=True) + urls_sources = urls['sources'] + if isinstance(urls_sources, dict): + urls_sources = urls_sources.values() + for source in urls_sources: + source_url = source.get('url') + if not source_url: + continue + protocol = source.get('protocol') + mimetype = source.get('mimetype') + if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m': + formats.extend(self._extract_f4m_formats( + source_url, clip_id, f4m_id='hds', fatal=False)) + elif mimetype == 'application/x-mpegURL': + formats.extend(self._extract_m3u8_formats( + source_url, clip_id, 'mp4', 'm3u8_native', + m3u8_id='hls', fatal=False)) + elif mimetype == 'application/dash+xml': + formats.extend(self._extract_mpd_formats( + source_url, clip_id, mpd_id='dash', fatal=False)) else: - formats.append({ - 'url': source_url, - 'tbr': tbr, - 'format_id': 'http%s' % ('-%d' % tbr if tbr else ''), - }) + tbr = fix_bitrate(source['bitrate']) + if protocol in ('rtmp', 'rtmpe'): + mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url) + if not mobj: + continue + path = mobj.group('path') + mp4colon_index = path.rfind('mp4:') + app = path[:mp4colon_index] + play_path = path[mp4colon_index:] + formats.append({ + 'url': '%s/%s' % (mobj.group('url'), app), + 'app': app, + 'play_path': play_path, + 'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf', + 'page_url': 'http://www.prosieben.de', + 'tbr': tbr, + 'ext': 'flv', + 'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''), + }) + else: + formats.append({ + 'url': source_url, + 'tbr': tbr, + 'format_id': 'http%s' % ('-%d' % tbr if tbr else ''), + }) self._sort_formats(formats) return { - 'duration': duration, + 'duration': float_or_none(video.get('duration')), 'formats': formats, } @@ -344,6 +384,11 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE): _TOKEN = 'prosieben' _SALT = '01!8d8F_)r9]4s[qeuXfP%' _CLIENT_NAME = 'kolibri-2.0.19-splec4' + + _ACCESS_ID = 'x_prosiebenmaxx-de' + _ENCRYPTION_KEY = 'Eeyeey9oquahthainoofashoyoikosag' + _IV = 'Aeluchoc6aevechuipiexeeboowedaok' + _CLIPID_REGEXES = [ r'"clip_id"\s*:\s+"(\d+)"', r'clipid: "(\d+)"', From 0c84002650cf21f38685a7446cd4cdb0f963a412 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Mon, 3 Jun 2019 12:43:39 +0100 Subject: [PATCH 28/46] [southpark] add support for videos with english audio track(closes #21271) --- youtube_dl/extractor/southpark.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py index da75a43a7..5374c8423 100644 --- a/youtube_dl/extractor/southpark.py +++ b/youtube_dl/extractor/southpark.py @@ -78,6 +78,10 @@ class SouthParkDeIE(SouthParkIE): 'only_matching': True, }] + def _real_initialize(self): + lang = self._get_cookies('http://www.southpark.de/alle-episoden/').get('SPS_video_language') + self._LANG = lang.value if lang else 'de' + class SouthParkNlIE(SouthParkIE): IE_NAME = 'southpark.nl' From c94c121a99ec893962aecbaf1f445dafc9e22bf1 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Tue, 4 Jun 2019 00:37:58 +0100 Subject: [PATCH 29/46] Revert "[southpark] add support for videos with english audio track" This reverts commit 0c84002650cf21f38685a7446cd4cdb0f963a412. --- youtube_dl/extractor/southpark.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py index 5374c8423..da75a43a7 100644 --- a/youtube_dl/extractor/southpark.py +++ b/youtube_dl/extractor/southpark.py @@ -78,10 +78,6 @@ class SouthParkDeIE(SouthParkIE): 'only_matching': True, }] - def _real_initialize(self): - lang = self._get_cookies('http://www.southpark.de/alle-episoden/').get('SPS_video_language') - self._LANG = lang.value if lang else 'de' - class SouthParkNlIE(SouthParkIE): IE_NAME = 'southpark.nl' From 59ca17b1c8351506fc10e7346ebd3fe238ddbf24 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Tue, 4 Jun 2019 21:03:52 +0100 Subject: [PATCH 30/46] [vvvvid] relax _VALID_URL(closes #21299) --- youtube_dl/extractor/vvvvid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py index 3d0dc403b..6906cd2ab 100644 --- a/youtube_dl/extractor/vvvvid.py +++ b/youtube_dl/extractor/vvvvid.py @@ -12,7 +12,7 @@ from ..utils import ( class VVVVIDIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?vvvvid\.it/#!(?:show|anime|film|series)/(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?vvvvid\.it/(?:#!)?(?:show|anime|film|series)/(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)' _TESTS = [{ # video_type == 'video/vvvvid' 'url': 'https://www.vvvvid.it/#!show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048/ping-pong', From 1a01639bf9514c20d54e7460ba9b493b3283ca9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Wed, 5 Jun 2019 03:06:35 +0700 Subject: [PATCH 31/46] [downloader/common] Improve rate limit (#21301) --- youtube_dl/downloader/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 646d7f779..1cdba89cd 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -176,7 +176,9 @@ class FileDownloader(object): return speed = float(byte_counter) / elapsed if speed > rate_limit: - time.sleep(max((byte_counter // rate_limit) - elapsed, 0)) + sleep_time = float(byte_counter) / rate_limit - elapsed + if sleep_time > 0: + time.sleep(sleep_time) def temp_name(self, filename): """Returns a temporary filename for the given filename.""" From 2e11e51c041bd95dd1305af5b958abacc27f39f7 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Wed, 5 Jun 2019 01:22:57 +0100 Subject: [PATCH 32/46] [cbsnews] fix extraction(closes #9659)(closes #15397) --- youtube_dl/extractor/cbsnews.py | 78 ++++++++++++++++++------------ youtube_dl/extractor/extractors.py | 1 + 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py index 51df15fac..345debcf0 100644 --- a/youtube_dl/extractor/cbsnews.py +++ b/youtube_dl/extractor/cbsnews.py @@ -1,40 +1,62 @@ # coding: utf-8 from __future__ import unicode_literals +import re +import zlib + from .common import InfoExtractor from .cbs import CBSIE +from ..compat import ( + compat_b64decode, + compat_urllib_parse_unquote, +) from ..utils import ( parse_duration, ) +class CBSNewsEmbedIE(CBSIE): + IE_NAME = 'cbsnews:embed' + _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/embed/video[^#]*#(?P<id>.+)' + _TESTS = [{ + 'url': 'https://www.cbsnews.com/embed/video/?v=1.c9b5b61492913d6660db0b2f03579ef25e86307a#1Vb7b9s2EP5XBAHbT6Gt98PAMKTJ0se6LVjWYWtdGBR1stlIpEBSTtwi%2F%2FvuJNkNhmHdGxgM2NL57vjd6zt%2B8PngdN%2Fyg79qeGvhzN%2FLGrS%2F%2BuBLB531V28%2B%2BO7Qg7%2Fy97r2z3xZ42NW8yLhDbA0S0KWlHnIijwKWJBHZZnHBa8Cgbpdf%2F89NM9Hi9fXifhpr8sr%2FlP848tn%2BTdXycX25zh4cdX%2FvHl6PmmPqnWQv9w8Ed%2B9GjYRim07bFEqdG%2BZVHuwTm65A7bVRrYtR5lAyMox7pigF6W4k%2By91mjspGsJ%2BwVae4%2BsvdnaO1p73HkXs%2FVisUDTGm7R8IcdnOROeq%2B19qT1amhA1VJtPenoTUgrtfKc9m7Rq8dP7nnjwOB7wg7ADdNt7VX64DWAWlKhPtmDEq22g4GF99x6Dk9E8OSsankHXqPNKDxC%2FdK7MLKTircTDgsI3mmj4OBdSq64dy7fd1x577RU1rt4cvMtOaulFYOd%2FLewRWvDO9lIgXFpZSnkZmjbv5SxKTPoQXClFbpsf%2Fhbbpzs0IB3vb8KkyzJQ%2BywOAgCrMpgRrz%2BKk4fvb7kFbR4XJCu0gAdtNO7woCwZTu%2BBUs9bam%2Fds71drVerpeisgrubLjAB4nnOSkWQnfr5W6o1ku5Xpr1MgrCbL0M0vUyDtfLLK15WiYp47xKWSLyjFVpwVmVJSLIoCjSOFkv3W7oKsVliwZJcB9nwXpZ5GEQQwY8jNKqKCBrgjTLeFxgdCIpazojDgnRtn43J6kG7nZ6cAbxh0EeFFk4%2B1u867cY5u4344n%2FxXjCqAjucdTHgLKojNKmSfO8KRsOFY%2FzKEYCKEJBzv90QA9nfm9gL%2BHulaFqUkz9ULUYxl62B3U%2FRVNLA8IhggaPycOoBuwOCESciDQVSSUgiOMsROB%2FhKfwCKOzEk%2B4k6rWd4uuT%2FwTDz7K7t3d3WLO8ISD95jSPQbayBacthbz86XVgxHwhex5zawzgDOmtp%2F3GPcXn0VXHdSS029%2Fj99UC%2FwJUvyKQ%2FzKyixIEVlYJOn4RxxuaH43Ty9fbJ5OObykHH435XAzJTHeOF4hhEUXD8URe%2FQ%2FBT%2BMpf8d5GN02Ox%2FfiGsl7TA7POu1xZ5%2BbTzcAVKMe48mqcC21hkacVEVScM26liVVBnrKkC4CLKyzAvHu0lhEaTKMFwI3a4SN9MsrfYzdBLq2vkwRD1gVviLT8kY9h2CHH6Y%2Bix6609weFtey4ESp60WtyeWMy%2BsmBuhsoKIyuoT%2Bq2R%2FrW5qi3g%2FvzS2j40DoixDP8%2BKP0yUdpXJ4l6Vla%2Bg9vce%2BC4yM5YlUcbA%2F0jLKdpmTwvsdN5z88nAIe08%2F0HgxeG1iv%2B6Hlhjh7uiW0SDzYNI92L401uha3JKYk268UVRzdOzNQvAaJqoXzAc80dAV440NZ1WVVAAMRYQ2KrGJFmDUsq8saWSnjvIj8t78y%2FRa3JRnbHVfyFpfwoDiGpPgjzekyUiKNlU3OMlwuLMmzgvEojllYVE2Z1HhImvsnk%2BuhusTEoB21PAtSFodeFK3iYhXEH9WOG2%2FkOE833sfeG%2Ff5cfHtEFNXgYes0%2FXj7aGivUgJ9XpusCtoNcNYVVnJVrrDo0OmJAutHCpuZul4W9lLcfy7BnuLPT02%2ByXsCTk%2B9zhzswIN04YueNSK%2BPtM0jS88QdLqSLJDTLsuGZJNolm2yO0PXh3UPnz9Ix5bfIAqxPjvETQsDCEiPG4QbqNyhBZISxybLnZYCrW5H3Axp690%2F0BJdXtDZ5ITuM4xj3f4oUHGzc5JeJmZKpp%2FjwKh4wMV%2FV1yx3emLoR0MwbG4K%2F%2BZgVep3PnzXGDHZ6a3i%2Fk%2BJrONDN13%2Bnq6tBTYk4o7cLGhBtqCC4KwacGHpEVuoH5JNro%2FE6JfE6d5RydbiR76k%2BW5wioDHBIjw1euhHjUGRB0y5A97KoaPx6MlL%2BwgboUVtUFRI%2FLemgTpdtF59ii7pab08kuPcfWzs0l%2FRI5takWnFpka0zOgWRtYcuf9aIxZMxlwr6IiGpsb6j2DQUXPl%2FimXI599Ev7fWjoPD78A', + 'only_matching': True, + }] + + def _real_extract(self, url): + item = self._parse_json(zlib.decompress(compat_b64decode( + compat_urllib_parse_unquote(self._match_id(url))), + -zlib.MAX_WBITS), None)['video']['items'][0] + return self._extract_video_info(item['mpxRefId'], 'cbsnews') + + class CBSNewsIE(CBSIE): IE_NAME = 'cbsnews' IE_DESC = 'CBS News' - _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)' + _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\da-z_-]+)' _TESTS = [ { # 60 minutes 'url': 'http://www.cbsnews.com/news/artificial-intelligence-positioned-to-be-a-game-changer/', 'info_dict': { - 'id': '_B6Ga3VJrI4iQNKsir_cdFo9Re_YJHE_', - 'ext': 'mp4', - 'title': 'Artificial Intelligence', - 'description': 'md5:8818145f9974431e0fb58a1b8d69613c', + 'id': 'Y_nf_aEg6WwO9OLAq0MpKaPgfnBUxfW4', + 'ext': 'flv', + 'title': 'Artificial Intelligence, real-life applications', + 'description': 'md5:a7aaf27f1b4777244de8b0b442289304', 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 1606, + 'duration': 317, 'uploader': 'CBSI-NEW', - 'timestamp': 1498431900, - 'upload_date': '20170625', + 'timestamp': 1476046464, + 'upload_date': '20161009', }, 'params': { - # m3u8 download + # rtmp download 'skip_download': True, }, }, { - 'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/', + 'url': 'https://www.cbsnews.com/video/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/', 'info_dict': { 'id': 'SNJBOYzXiWBOvaLsdzwH8fmtP1SCd91Y', 'ext': 'mp4', @@ -60,37 +82,29 @@ class CBSNewsIE(CBSIE): # 48 hours 'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/', 'info_dict': { - 'id': 'QpM5BJjBVEAUFi7ydR9LusS69DPLqPJ1', - 'ext': 'mp4', 'title': 'Cold as Ice', - 'description': 'Can a childhood memory of a friend\'s murder solve a 1957 cold case? "48 Hours" correspondent Erin Moriarty has the latest.', - 'upload_date': '20170604', - 'timestamp': 1496538000, - 'uploader': 'CBSI-NEW', - }, - 'params': { - 'skip_download': True, + 'description': 'Can a childhood memory solve the 1957 murder of 7-year-old Maria Ridulph?', }, + 'playlist_mincount': 7, }, ] def _real_extract(self, url): - video_id = self._match_id(url) + display_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + webpage = self._download_webpage(url, display_id) - video_info = self._parse_json(self._html_search_regex( - r'(?:<ul class="media-list items" id="media-related-items"[^>]*><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'', - webpage, 'video JSON info', default='{}'), video_id, fatal=False) - - if video_info: - item = video_info['item'] if 'item' in video_info else video_info - else: - state = self._parse_json(self._search_regex( - r'data-cbsvideoui-options=(["\'])(?P<json>{.+?})\1', webpage, - 'playlist JSON info', group='json'), video_id)['state'] - item = state['playlist'][state['pid']] + entries = [] + for embed_url in re.findall(r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage): + entries.append(self.url_result(embed_url, CBSNewsEmbedIE.ie_key())) + if entries: + return self.playlist_result( + entries, playlist_title=self._html_search_meta(['og:title', 'twitter:title'], webpage), + playlist_description=self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage)) + item = self._parse_json(self._html_search_regex( + r'CBSNEWS\.defaultPayload\s*=\s*({.+})', + webpage, 'video JSON info'), display_id)['items'][0] return self._extract_video_info(item['mpxRefId'], 'cbsnews') diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index eb5efd1e8..711d3f402 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -173,6 +173,7 @@ from .cbs import CBSIE from .cbslocal import CBSLocalIE from .cbsinteractive import CBSInteractiveIE from .cbsnews import ( + CBSNewsEmbedIE, CBSNewsIE, CBSNewsLiveVideoIE, ) From 4c78c3d70086f9f8ea7ddcf8896065443c6dc37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 7 Jun 2019 22:58:19 +0700 Subject: [PATCH 33/46] [bitchute] Extract HTML5 formats (closes #21306) --- youtube_dl/extractor/bitchute.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py index 1d69dafbd..430663fbf 100644 --- a/youtube_dl/extractor/bitchute.py +++ b/youtube_dl/extractor/bitchute.py @@ -55,6 +55,11 @@ class BitChuteIE(InfoExtractor): formats = [ {'url': format_url} for format_url in orderedSet(format_urls)] + + if not formats: + formats = self._parse_html5_media_entries( + url, webpage, video_id)[0]['formats'] + self._check_formats(formats, video_id) self._sort_formats(formats) From bf3c9326634f4a918d0336b2c5f104ac492b5510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 7 Jun 2019 23:13:33 +0700 Subject: [PATCH 34/46] [youtube] Fix average rating extraction (closes #21304) --- youtube_dl/extractor/youtube.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 5f1957a59..963c73a2d 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2224,6 +2224,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage, 'view count', default=None)) + average_rating = ( + float_or_none(video_details.get('averageRating')) + or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0]))) + # subtitles video_subtitles = self.extract_subtitles(video_id, video_webpage) automatic_captions = self.extract_automatic_captions(video_id, video_webpage) @@ -2353,7 +2357,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'view_count': view_count, 'like_count': like_count, 'dislike_count': dislike_count, - 'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]), + 'average_rating': average_rating, 'formats': formats, 'is_live': is_live, 'start_time': start_time, From 2efefddafd1c83a900b091c0051f84a1ba2fcbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Fri, 7 Jun 2019 23:48:57 +0700 Subject: [PATCH 35/46] [redbulltv:rrn:content] Add extractor (closes #21297) --- youtube_dl/extractor/extractors.py | 5 ++++- youtube_dl/extractor/redbulltv.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 711d3f402..b1ed8a4b2 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -938,7 +938,10 @@ from .raywenderlich import ( ) from .rbmaradio import RBMARadioIE from .rds import RDSIE -from .redbulltv import RedBullTVIE +from .redbulltv import ( + RedBullTVIE, + RedBullTVRrnContentIE, +) from .reddit import ( RedditIE, RedditRIE, diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py index 5a03e8e11..dbe1aaded 100644 --- a/youtube_dl/extractor/redbulltv.py +++ b/youtube_dl/extractor/redbulltv.py @@ -104,3 +104,25 @@ class RedBullTVIE(InfoExtractor): 'formats': formats, 'subtitles': subtitles, } + + +class RedBullTVRrnContentIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com(?:/[^/]+)?(?:/tv)?)/(?:video|live)/rrn:content:[^:]+:(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})' + _TESTS = [{ + 'url': 'https://www.redbull.com/int-en/tv/video/rrn:content:live-videos:e3e6feb4-e95f-50b7-962a-c70f8fd13c73/mens-dh-finals-fort-william', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/tv/video/rrn:content:videos:a36a0f36-ff1b-5db8-a69d-ee11a14bf48b/tn-ts-style?playlist=rrn:content:event-profiles:83f05926-5de8-5389-b5e4-9bb312d715e8:extras', + 'only_matching': True, + }] + + def _real_extract(self, url): + display_id = self._match_id(url) + + webpage = self._download_webpage(url, display_id) + + video_url = self._og_search_url(webpage) + + return self.url_result( + video_url, ie=RedBullTVIE.ie_key(), + video_id=RedBullTVIE._match_id(video_url)) From e75220b11af01fcee6ca40847a1b94ccd0bce8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 8 Jun 2019 00:45:28 +0700 Subject: [PATCH 36/46] [tvnow] Extract HD formats (closes #21201) --- youtube_dl/extractor/tvnow.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/tvnow.py b/youtube_dl/extractor/tvnow.py index ecd0fb937..9c8a8a0dc 100644 --- a/youtube_dl/extractor/tvnow.py +++ b/youtube_dl/extractor/tvnow.py @@ -47,15 +47,23 @@ class TVNowBaseIE(InfoExtractor): r'\.ism/(?:[^.]*\.(?:m3u8|mpd)|[Mm]anifest)', '.ism/' + suffix, manifest_url)) - formats = self._extract_mpd_formats( - url_repl('dash', '.mpd'), video_id, - mpd_id='dash', fatal=False) - formats.extend(self._extract_ism_formats( - url_repl('hss', 'Manifest'), - video_id, ism_id='mss', fatal=False)) - formats.extend(self._extract_m3u8_formats( - url_repl('hls', '.m3u8'), video_id, 'mp4', - 'm3u8_native', m3u8_id='hls', fatal=False)) + def make_urls(proto, suffix): + urls = [url_repl(proto, suffix)] + hd_url = urls[0].replace('/manifest/', '/ngvod/') + if hd_url != urls[0]: + urls.append(hd_url) + return urls + + for man_url in make_urls('dash', '.mpd'): + formats = self._extract_mpd_formats( + man_url, video_id, mpd_id='dash', fatal=False) + for man_url in make_urls('hss', 'Manifest'): + formats.extend(self._extract_ism_formats( + man_url, video_id, ism_id='mss', fatal=False)) + for man_url in make_urls('hls', '.m3u8'): + formats.extend(self._extract_m3u8_formats( + man_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', + fatal=False)) if formats: break else: From f4c99cd6354a050f545b1404f51106468a1516d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= <sedivy.miro@gmail.com> Date: Fri, 7 Jun 2019 21:03:58 +0200 Subject: [PATCH 37/46] [openload] Add support for oload.website (#21329) --- youtube_dl/extractor/openload.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index b96be6f64..32d546e4e 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -244,7 +244,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|press|pw|live|space|services)|oladblock\.(?:services|xyz|me)|openloed\.co)' + _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|press|pw|live|space|services|website)|oladblock\.(?:services|xyz|me)|openloed\.co)' _VALID_URL = r'''(?x) https?:// (?P<host> @@ -360,6 +360,9 @@ class OpenloadIE(InfoExtractor): }, { 'url': 'https://oload.press/embed/drTBl1aOTvk/', 'only_matching': True, + }, { + 'url': 'https://oload.website/embed/drTBl1aOTvk/', + 'only_matching': True, }, { 'url': 'https://oladblock.services/f/b8NWEgkqNLI/', 'only_matching': True, From 01b517a20a2645a05b44d3ae65b587fef3fda788 Mon Sep 17 00:00:00 2001 From: biwubo <biwubo> Date: Fri, 24 May 2019 14:55:20 +0000 Subject: [PATCH 38/46] [vlive] Add CH+ support (closes #16887) --- youtube_dl/extractor/vlive.py | 86 +++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py index d68f222e7..78436c45c 100644 --- a/youtube_dl/extractor/vlive.py +++ b/youtube_dl/extractor/vlive.py @@ -24,6 +24,7 @@ from ..utils import ( class VLiveIE(InfoExtractor): IE_NAME = 'vlive' _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)' + _NETRC_MACHINE = 'vlive' _TESTS = [{ 'url': 'http://www.vlive.tv/video/1326', 'md5': 'cc7314812855ce56de70a06a27314983', @@ -47,12 +48,59 @@ class VLiveIE(InfoExtractor): 'params': { 'skip_download': True, }, + }, { + 'url': 'https://www.vlive.tv/video/129100', + 'md5': 'ca2569453b79d66e5b919e5d308bff6b', + 'info_dict': { + 'id': '129100', + 'ext': 'mp4', + 'title': "[V LIVE] [BTS+] Run BTS! 2019 - EP.71 :: Behind the scene", + 'creator': "BTS+", + 'view_count': int, + 'subtitles': 'mincount:10', + }, + 'skip': 'This video is only available for CH+ subscribers', }] @classmethod def suitable(cls, url): return False if VLivePlaylistIE.suitable(url) else super(VLiveIE, cls).suitable(url) + def _real_initialize(self): + self._login() + + def _login(self): + email, password = self._get_login_info() + if None in (email, password): + return + + def is_logged_in(): + login_info = self._download_json( + 'https://www.vlive.tv/auth/loginInfo', None, + note='Downloading login info', + headers={'Referer': 'https://www.vlive.tv/home'}) + + return try_get(login_info, + lambda x: x['message']['login'], bool) or False + + if is_logged_in(): + return + + LOGIN_URL = 'https://www.vlive.tv/auth/email/login' + self._request_webpage(LOGIN_URL, None, + note='Downloading login cookies') + + self._download_webpage( + LOGIN_URL, None, note='Logging in', + data=urlencode_postdata({'email': email, 'pwd': password}), + headers={ + 'Referer': LOGIN_URL, + 'Content-Type': 'application/x-www-form-urlencoded' + }) + + if not is_logged_in(): + raise ExtractorError('Unable to log in', expected=True) + def _real_extract(self, url): video_id = self._match_id(url) @@ -77,10 +125,7 @@ class VLiveIE(InfoExtractor): if status in ('LIVE_ON_AIR', 'BIG_EVENT_ON_AIR'): return self._live(video_id, webpage) elif status in ('VOD_ON_AIR', 'BIG_EVENT_INTRO'): - if long_video_id and key: - return self._replay(video_id, webpage, long_video_id, key) - else: - status = 'COMING_SOON' + return self._replay(video_id, webpage, long_video_id, key) if status == 'LIVE_END': raise ExtractorError('Uploading for replay. Please wait...', @@ -91,13 +136,15 @@ class VLiveIE(InfoExtractor): raise ExtractorError('We are sorry, ' 'but the live broadcast has been canceled.', expected=True) + elif status == 'ONLY_APP': + raise ExtractorError('Unsupported video type', expected=True) else: raise ExtractorError('Unknown status %s' % status) def _get_common_fields(self, webpage): title = self._og_search_title(webpage) creator = self._html_search_regex( - r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)', + r'<div[^>]+class="info_area"[^>]*>\s*(?:<em[^>]*>.*</em\s*>\s*)?<a\s+[^>]*>([^<]+)', webpage, 'creator', fatal=False) thumbnail = self._og_search_thumbnail(webpage) return { @@ -107,14 +154,7 @@ class VLiveIE(InfoExtractor): } def _live(self, video_id, webpage): - init_page = self._download_webpage( - 'https://www.vlive.tv/video/init/view', - video_id, note='Downloading live webpage', - data=urlencode_postdata({'videoSeq': video_id}), - headers={ - 'Referer': 'https://www.vlive.tv/video/%s' % video_id, - 'Content-Type': 'application/x-www-form-urlencoded' - }) + init_page = self._download_init_page(video_id) live_params = self._search_regex( r'"liveStreamInfo"\s*:\s*(".*"),', @@ -140,6 +180,16 @@ class VLiveIE(InfoExtractor): return info def _replay(self, video_id, webpage, long_video_id, key): + if '' in (long_video_id, key): + init_page = self._download_init_page(video_id) + video_info = self._parse_json(self._search_regex( + r'(?s)oVideoStatus\s*=\s*({.*})', init_page, 'video info'), + video_id) + if video_info['status'] == 'NEED_CHANNEL_PLUS': + self.raise_login_required( + 'This video is only available for CH+ subscribers') + long_video_id, key = video_info['vid'], video_info['inkey'] + playinfo = self._download_json( 'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s' % compat_urllib_parse_urlencode({ @@ -180,6 +230,16 @@ class VLiveIE(InfoExtractor): }) return info + def _download_init_page(self, video_id): + return self._download_webpage( + 'https://www.vlive.tv/video/init/view', + video_id, note='Downloading live webpage', + data=urlencode_postdata({'videoSeq': video_id}), + headers={ + 'Referer': 'https://www.vlive.tv/video/%s' % video_id, + 'Content-Type': 'application/x-www-form-urlencoded' + }) + class VLiveChannelIE(InfoExtractor): IE_NAME = 'vlive:channel' From ef19739e64e91bb5ca2eeccd3ff6fdefb1a98cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 8 Jun 2019 02:19:07 +0700 Subject: [PATCH 39/46] [vlive] Improve and style (closes #21209) --- youtube_dl/extractor/vlive.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py index 78436c45c..f75d4e254 100644 --- a/youtube_dl/extractor/vlive.py +++ b/youtube_dl/extractor/vlive.py @@ -79,16 +79,12 @@ class VLiveIE(InfoExtractor): 'https://www.vlive.tv/auth/loginInfo', None, note='Downloading login info', headers={'Referer': 'https://www.vlive.tv/home'}) - - return try_get(login_info, - lambda x: x['message']['login'], bool) or False - - if is_logged_in(): - return + return try_get( + login_info, lambda x: x['message']['login'], bool) or False LOGIN_URL = 'https://www.vlive.tv/auth/email/login' - self._request_webpage(LOGIN_URL, None, - note='Downloading login cookies') + self._request_webpage( + LOGIN_URL, None, note='Downloading login cookies') self._download_webpage( LOGIN_URL, None, note='Logging in', @@ -144,7 +140,7 @@ class VLiveIE(InfoExtractor): def _get_common_fields(self, webpage): title = self._og_search_title(webpage) creator = self._html_search_regex( - r'<div[^>]+class="info_area"[^>]*>\s*(?:<em[^>]*>.*</em\s*>\s*)?<a\s+[^>]*>([^<]+)', + r'<div[^>]+class="info_area"[^>]*>\s*(?:<em[^>]*>.*?</em\s*>\s*)?<a\s+[^>]*>([^<]+)', webpage, 'creator', fatal=False) thumbnail = self._og_search_thumbnail(webpage) return { @@ -183,9 +179,10 @@ class VLiveIE(InfoExtractor): if '' in (long_video_id, key): init_page = self._download_init_page(video_id) video_info = self._parse_json(self._search_regex( - r'(?s)oVideoStatus\s*=\s*({.*})', init_page, 'video info'), + (r'(?s)oVideoStatus\s*=\s*({.+?})\s*</script', + r'(?s)oVideoStatus\s*=\s*({.+})'), init_page, 'video info'), video_id) - if video_info['status'] == 'NEED_CHANNEL_PLUS': + if video_info.get('status') == 'NEED_CHANNEL_PLUS': self.raise_login_required( 'This video is only available for CH+ subscribers') long_video_id, key = video_info['vid'], video_info['inkey'] From 178663df528dddb1b628b70e4ad167cc97a8ada5 Mon Sep 17 00:00:00 2001 From: biwubo <45994985+biwubo@users.noreply.github.com> Date: Fri, 7 Jun 2019 20:40:47 +0100 Subject: [PATCH 40/46] [vlive:playlist] Redirect to video extractor when no playlist is found (#20590) --- youtube_dl/extractor/vlive.py | 57 ++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py index f75d4e254..6bfbceef2 100644 --- a/youtube_dl/extractor/vlive.py +++ b/youtube_dl/extractor/vlive.py @@ -332,26 +332,43 @@ class VLiveChannelIE(InfoExtractor): class VLivePlaylistIE(InfoExtractor): IE_NAME = 'vlive:playlist' _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<video_id>[0-9]+)/playlist/(?P<id>[0-9]+)' - _TEST = { + _VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s' + _TESTS = [{ + 'url': 'https://www.vlive.tv/video/117956/playlist/117963', + 'info_dict': { + 'id': '117963', + 'title': '아이돌룸(IDOL ROOM) 41회 - (여자)아이들' + }, + 'playlist_mincount': 10 + }, { 'url': 'http://www.vlive.tv/video/22867/playlist/22912', 'info_dict': { - 'id': '22912', - 'title': 'Valentine Day Message from TWICE' + 'id': '22867', + 'ext': 'mp4', + 'title': '[V LIVE] Valentine Day Message from MINA', + 'creator': "TWICE", + 'view_count': int }, - 'playlist_mincount': 9 - } + 'params': { + 'skip_download': True, + } + }] + + def _build_video_result(self, video_id, message): + self.to_screen(message) + return self.url_result( + self._VIDEO_URL_TEMPLATE % video_id, + ie=VLiveIE.ie_key(), video_id=video_id) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id, playlist_id = mobj.group('video_id', 'id') - VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s' if self._downloader.params.get('noplaylist'): - self.to_screen( - 'Downloading just video %s because of --no-playlist' % video_id) - return self.url_result( - VIDEO_URL_TEMPLATE % video_id, - ie=VLiveIE.ie_key(), video_id=video_id) + return self._build_video_result( + video_id, + 'Downloading just video %s because of --no-playlist' + % video_id) self.to_screen( 'Downloading playlist %s - add --no-playlist to just download video' @@ -361,15 +378,21 @@ class VLivePlaylistIE(InfoExtractor): 'http://www.vlive.tv/video/%s/playlist/%s' % (video_id, playlist_id), playlist_id) - item_ids = self._parse_json( - self._search_regex( - r'playlistVideoSeqs\s*=\s*(\[[^]]+\])', webpage, - 'playlist video seqs'), - playlist_id) + raw_item_ids = self._search_regex( + r'playlistVideoSeqs\s*=\s*(\[[^]]+\])', webpage, + 'playlist video seqs', default=None, fatal=False) + + if not raw_item_ids: + return self._build_video_result( + video_id, + 'Downloading just video %s because no playlist was found' + % video_id) + + item_ids = self._parse_json(raw_item_ids, playlist_id) entries = [ self.url_result( - VIDEO_URL_TEMPLATE % item_id, ie=VLiveIE.ie_key(), + self._VIDEO_URL_TEMPLATE % item_id, ie=VLiveIE.ie_key(), video_id=compat_str(item_id)) for item_id in item_ids] From 4831ef7fe41cf4dfca5957c61635fb5a547ad9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 8 Jun 2019 02:42:01 +0700 Subject: [PATCH 41/46] [vlive] Add comments to tests and quotes consistency --- youtube_dl/extractor/vlive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py index 6bfbceef2..c3429f723 100644 --- a/youtube_dl/extractor/vlive.py +++ b/youtube_dl/extractor/vlive.py @@ -54,8 +54,8 @@ class VLiveIE(InfoExtractor): 'info_dict': { 'id': '129100', 'ext': 'mp4', - 'title': "[V LIVE] [BTS+] Run BTS! 2019 - EP.71 :: Behind the scene", - 'creator': "BTS+", + 'title': '[V LIVE] [BTS+] Run BTS! 2019 - EP.71 :: Behind the scene', + 'creator': 'BTS+', 'view_count': int, 'subtitles': 'mincount:10', }, @@ -334,6 +334,7 @@ class VLivePlaylistIE(InfoExtractor): _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<video_id>[0-9]+)/playlist/(?P<id>[0-9]+)' _VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s' _TESTS = [{ + # regular working playlist 'url': 'https://www.vlive.tv/video/117956/playlist/117963', 'info_dict': { 'id': '117963', @@ -341,12 +342,13 @@ class VLivePlaylistIE(InfoExtractor): }, 'playlist_mincount': 10 }, { + # playlist with no playlistVideoSeqs 'url': 'http://www.vlive.tv/video/22867/playlist/22912', 'info_dict': { 'id': '22867', 'ext': 'mp4', 'title': '[V LIVE] Valentine Day Message from MINA', - 'creator': "TWICE", + 'creator': 'TWICE', 'view_count': int }, 'params': { From c2ee6fa66ac082a74e645e605c346d0abe95afe8 Mon Sep 17 00:00:00 2001 From: biwubo <biwubo> Date: Thu, 9 May 2019 18:11:27 +0000 Subject: [PATCH 42/46] [ted] Fix playlist extraction (closes #20844) --- youtube_dl/extractor/ted.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index 645942dfd..17dc41a39 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -5,8 +5,12 @@ import re from .common import InfoExtractor -from ..compat import compat_str +from ..compat import ( + compat_str, + compat_urlparse +) from ..utils import ( + extract_attributes, float_or_none, int_or_none, try_get, @@ -20,7 +24,7 @@ class TEDIE(InfoExtractor): (?P<proto>https?://) (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/ ( - (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist + (?P<type_playlist>playlists(?:/(?P<playlist_id>\d+))?) # We have a playlist | ((?P<type_talk>talks)) # We have a simple talk | @@ -84,6 +88,7 @@ class TEDIE(InfoExtractor): 'info_dict': { 'id': '10', 'title': 'Who are the hackers?', + 'description': 'md5:49a0dbe8fb76d81a0e64b4a80af7f15a' }, 'playlist_mincount': 6, }, { @@ -150,22 +155,19 @@ class TEDIE(InfoExtractor): webpage = self._download_webpage(url, name, 'Downloading playlist webpage') - info = self._extract_info(webpage) - playlist_info = try_get( - info, lambda x: x['__INITIAL_DATA__']['playlist'], - dict) or info['playlist'] + playlist_entries = [] + for entry in re.findall(r'(?s)<[^>]+data-ga-context="playlist"[^>]*>', webpage): + attrs = extract_attributes(entry) + entry_url = compat_urlparse.urljoin(url, attrs['href']) + playlist_entries.append(self.url_result(entry_url, self.ie_key())) - playlist_entries = [ - self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key()) - for talk in try_get( - info, lambda x: x['__INITIAL_DATA__']['talks'], - dict) or info['talks'] - ] + final_url = self._og_search_url(webpage) return self.playlist_result( playlist_entries, - playlist_id=compat_str(playlist_info['id']), - playlist_title=playlist_info['title']) + playlist_id=re.match(self._VALID_URL, final_url, re.VERBOSE).group('playlist_id'), + playlist_title=self._og_search_title(webpage), + playlist_description=self._og_search_description(webpage)) def _talk_info(self, url, video_name): webpage = self._download_webpage(url, video_name) From dbb18861148beed6630ebf895cd3f9e8165f75df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 8 Jun 2019 03:06:41 +0700 Subject: [PATCH 43/46] [ted] Improve playlist extraction (closes #21032) --- youtube_dl/extractor/ted.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index 17dc41a39..9b60cc462 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -157,16 +157,19 @@ class TEDIE(InfoExtractor): 'Downloading playlist webpage') playlist_entries = [] - for entry in re.findall(r'(?s)<[^>]+data-ga-context="playlist"[^>]*>', webpage): + for entry in re.findall(r'(?s)<[^>]+data-ga-context=["\']playlist["\'][^>]*>', webpage): attrs = extract_attributes(entry) entry_url = compat_urlparse.urljoin(url, attrs['href']) playlist_entries.append(self.url_result(entry_url, self.ie_key())) - final_url = self._og_search_url(webpage) + final_url = self._og_search_url(webpage, fatal=False) + playlist_id = ( + re.match(self._VALID_URL, final_url).group('playlist_id') + if final_url else None) + return self.playlist_result( - playlist_entries, - playlist_id=re.match(self._VALID_URL, final_url, re.VERBOSE).group('playlist_id'), - playlist_title=self._og_search_title(webpage), + playlist_entries, playlist_id=playlist_id, + playlist_title=self._og_search_title(webpage, fatal=False), playlist_description=self._og_search_description(webpage)) def _talk_info(self, url, video_name): From e35fc5ebc7eef38f8b354259849084f7a7b5ebb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 8 Jun 2019 03:45:31 +0700 Subject: [PATCH 44/46] [ChangeLog] Actualize [ci skip] --- ChangeLog | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ChangeLog b/ChangeLog index 3babb6f48..c49e825d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,41 @@ +version <unreleased> + +Core +* [downloader/common] Improve rate limit (#21301) +* [utils] Improve strip_or_none +* [extractor/common] Strip src attribute for HTML5 entries code (#18485, + #21169) + +Extractors +* [ted] Fix playlist extraction (#20844, #21032) +* [vlive:playlist] Fix video extraction when no playlist is found (#20590) ++ [vlive] Add CH+ support (#16887, #21209) ++ [openload] Add support for oload.website (#21329) ++ [tvnow] Extract HD formats (#21201) ++ [redbulltv] Add support for rrn:content URLs (#21297) +* [youtube] Fix average rating extraction (#21304) ++ [bitchute] Extract HTML5 formats (#21306) +* [cbsnews] Fix extraction (#9659, #15397) +* [vvvvid] Relax URL regular expression (#21299) ++ [prosiebensat1] Add support for new API (#21272) ++ [vrv] Extract adaptive_hls formats (#21243) +* [viki] Switch to HTTPS (#21001) +* [LiveLeak] Check if the original videos exist (#21206, #21208) +* [rtp] Fix extraction (#15099) +* [youtube] Improve DRM protected videos detection (#1774) ++ [srgssrplay] Add support for popupvideoplayer URLs (#21155) ++ [24video] Add support for porno.24video.net (#21194) ++ [24video] Add support for 24video.site (#21193) +- [pornflip] Remove extractor +- [criterion] Remove extractor (#21195) +* [pornhub] Use HTTPS (#21061) +* [bitchute] Fix uploader extraction (#21076) +* [streamcloud] Reduce waiting time to 6 seconds (#21092) +- [novamov] Remove extractors (#21077) ++ [openload] Add support for oload.press (#21135) +* [vivo] Fix extraction (#18906, #19217) + + version 2019.05.20 Core From 3b2fd09596f4a1efd79c9e6193c3032adbd89da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sat, 8 Jun 2019 03:47:30 +0700 Subject: [PATCH 45/46] release 2019.06.08 --- .github/ISSUE_TEMPLATE/1_broken_site.md | 6 +++--- .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++-- .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++-- .github/ISSUE_TEMPLATE/4_bug_report.md | 6 +++--- .github/ISSUE_TEMPLATE/5_feature_request.md | 4 ++-- ChangeLog | 2 +- docs/supportedsites.md | 9 ++------- youtube_dl/version.py | 2 +- 8 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index dc303946e..104ad598c 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' <!-- Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.05.20. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.06.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape. - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2019.05.20** +- [ ] I've verified that I'm running youtube-dl version **2019.06.08** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.05.20 + [debug] youtube-dl version 2019.06.08 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index 46e143c8a..f711af040 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' <!-- Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.05.20. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.06.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights. - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2019.05.20** +- [ ] I've verified that I'm running youtube-dl version **2019.06.08** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index bc6c4694b..ae865a8b0 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' <!-- Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.05.20. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.06.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2019.05.20** +- [ ] I've verified that I'm running youtube-dl version **2019.06.08** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index bcc51f986..8246b570e 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' <!-- Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.05.20. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.06.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape. - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2019.05.20** +- [ ] I've verified that I'm running youtube-dl version **2019.06.08** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.05.20 + [debug] youtube-dl version 2019.06.08 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index c8d16960e..292c2e697 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' <!-- Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.05.20. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2019.06.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2019.05.20** +- [ ] I've verified that I'm running youtube-dl version **2019.06.08** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/ChangeLog b/ChangeLog index c49e825d0..c4d485ff1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version <unreleased> +version 2019.06.08 Core * [downloader/common] Improve rate limit (#21301) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 404a2f0a4..bfd15b4dc 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -78,7 +78,6 @@ - **AudioBoom** - **audiomack** - **audiomack:album** - - **auroravid**: AuroraVid - **AWAAN** - **awaan:live** - **awaan:season** @@ -150,6 +149,7 @@ - **CBSInteractive** - **CBSLocal** - **cbsnews**: CBS News + - **cbsnews:embed** - **cbsnews:livevideo**: CBS News Live Videos - **CBSSports** - **CCMA** @@ -174,7 +174,6 @@ - **Clipsyndicate** - **CloserToTruth** - **CloudflareStream** - - **cloudtime**: CloudTime - **Cloudy** - **Clubic** - **Clyp** @@ -194,7 +193,6 @@ - **Coub** - **Cracked** - **Crackle** - - **Criterion** - **CrooksAndLiars** - **crunchyroll** - **crunchyroll:playlist** @@ -609,7 +607,6 @@ - **nowness** - **nowness:playlist** - **nowness:series** - - **nowvideo**: NowVideo - **Noz** - **npo**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **npo.nl:live** @@ -693,7 +690,6 @@ - **PopcornTV** - **PornCom** - **PornerBros** - - **PornFlip** - **PornHd** - **PornHub**: PornHub and Thumbzilla - **PornHubPlaylist** @@ -734,6 +730,7 @@ - **RBMARadio** - **RDS**: RDS.ca - **RedBullTV** + - **RedBullTVRrnContent** - **Reddit** - **RedditR** - **RedTube** @@ -1024,7 +1021,6 @@ - **videomore:video** - **VideoPremium** - **VideoPress** - - **videoweed**: VideoWeed - **Vidio** - **VidLii** - **vidme** @@ -1101,7 +1097,6 @@ - **Weibo** - **WeiboMobile** - **WeiqiTV**: WQTV - - **wholecloud**: WholeCloud - **Wimp** - **Wistia** - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 8df77378b..6aa666bc9 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.05.20' +__version__ = '2019.06.08' From 35c2dd48d9c61015627b00c2081b953fdea03f43 Mon Sep 17 00:00:00 2001 From: Remita Amine <remitamine@gmail.com> Date: Sun, 9 Jun 2019 00:12:51 +0100 Subject: [PATCH 46/46] [cbs] improve DRM protected videos detection(closes #21339) --- youtube_dl/extractor/cbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py index 376db7263..4a19a73d2 100644 --- a/youtube_dl/extractor/cbs.py +++ b/youtube_dl/extractor/cbs.py @@ -69,7 +69,7 @@ class CBSIE(CBSBaseIE): last_e = None for item in items_data.findall('.//item'): asset_type = xpath_text(item, 'assetType') - if not asset_type or asset_type in asset_types or asset_type in ('HLS_FPS', 'DASH_CENC'): + if not asset_type or asset_type in asset_types or 'HLS_FPS' in asset_type or 'DASH_CENC' in asset_type: continue asset_types.append(asset_type) query = {