Merge remote-tracking branch 'upstream/master' into porntrex

2019-04-18 16:01:14 +03:00 · 2019-04-18 16:01:14 +03:00 · c5fd4cd5be
commit c5fd4cd5be
parent e7a76dde02 f3914b06a0
7 changed files with 85 additions and 25 deletions
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@ -6,8 +6,8 @@

 ---

-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.07*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.17*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.17**

 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections
@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.04.07
+[debug] youtube-dl version 2019.04.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
--- a/25
+++ b/25
@ -1,3 +1,28 @@
+version 2019.04.17
+
+Extractors
+* [openload] Randomize User-Agent (closes #20688)
+ [openload] Add support for oladblock domains (#20471)
+* [adn] Fix subtitle extraction (#12724)
+ [aol] Add support for localized websites
+ [yahoo] Add support GYAO episode URLs
+ [yahoo] Add support for streaming.yahoo.co.jp (#5811, #7098)
+ [yahoo] Add support for gyao.yahoo.co.jp
+* [aenetworks] Fix history topic extraction and extract more formats
+ [cbs] Extract smpte and vtt subtitles
+ [streamango] Add support for streamcherry.com (#20592)
+ [yourporn] Add support for sxyprn.com (#20646)
+* [mgtv] Fix extraction (#20650)
+* [linkedin:learning] Use urljoin for form action URL (#20431)
+ [gdc] Add support for kaltura embeds (#20575)
+* [dispeak] Improve mp4 bitrate extraction
+* [kaltura] Sanitize embed URLs
+* [jwplatfom] Do not match manifest URLs (#20596)
+* [aol] Restrict URL regular expression and improve format extraction
+ [tiktok] Add support for new URL schema (#20573)
+ [stv:player] Add support for player.stv.tv (#20586)
+
+
 version 2019.04.07

 Core
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@ -46,6 +46,7 @@
 - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
 - **AnimeOnDemand**
 - **Anvato**
+ - **aol.com**
 - **APA**
 - **Aparat**
 - **AppleConnect**
@ -632,7 +633,6 @@
 - **OdaTV**
 - **Odnoklassniki**
 - **OktoberfestTV**
- - **on.aol.com**
 - **OnDemandKorea**
 - **onet.pl**
 - **onet.tv**
@ -853,6 +853,7 @@
 - **StreamCZ**
 - **StreetVoice**
 - **StretchInternet**
+ - **stv:player**
 - **SunPorno**
 - **SVT**
 - **SVTPage**
@ -1124,6 +1125,8 @@
 - **XVideos**
 - **XXXYMovies**
 - **Yahoo**: Yahoo screen and movies
+ - **yahoo:gyao**
+ - **yahoo:gyao:player**
 - **YandexDisk**
 - **yandexmusic:album**: Яндекс.Музыка - Альбом
 - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@ -60,14 +60,19 @@ class ADNIE(InfoExtractor):

        enc_subtitles = self._download_webpage(
            urljoin(self._BASE_URL, sub_path),
-            video_id, 'Downloading subtitles data', fatal=False)
+            video_id, 'Downloading subtitles location', fatal=False) or '{}'
+        subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
+        if subtitle_location:
+            enc_subtitles = self._download_webpage(
+                urljoin(self._BASE_URL, subtitle_location),
+                video_id, 'Downloading subtitles data', fatal=False)
        if not enc_subtitles:
            return None

        # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
        dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
            bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(binascii.unhexlify(self._K + '083db5aebd9353b4')),
+            bytes_to_intlist(binascii.unhexlify(self._K + '4421de0a5f0814ba')),
            bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
        ))
        subtitles_json = self._parse_json(
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals

+import re
+
 from .adobepass import AdobePassIE
 from ..utils import (
    smuggle_url,
@ -12,16 +14,16 @@ from ..utils import (
 class BravoTVIE(AdobePassIE):
    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
    _TESTS = [{
-        'url': 'http://www.bravotv.com/last-chance-kitchen/season-5/videos/lck-ep-12-fishy-finale',
-        'md5': '9086d0b7ef0ea2aabc4781d75f4e5863',
+        'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
+        'md5': 'e34684cfea2a96cd2ee1ef3a60909de9',
        'info_dict': {
-            'id': 'zHyk1_HU_mPy',
+            'id': 'epL0pmK1kQlT',
            'ext': 'mp4',
-            'title': 'LCK Ep 12: Fishy Finale',
-            'description': 'S13/E12: Two eliminated chefs have just 12 minutes to cook up a delicious fish dish.',
+            'title': 'The Top Chef Season 16 Winner Is...',
+            'description': 'Find out who takes the title of Top Chef!',
            'uploader': 'NBCU-BRAV',
-            'upload_date': '20160302',
-            'timestamp': 1456945320,
+            'upload_date': '20190314',
+            'timestamp': 1552591860,
        }
    }, {
        'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
@ -32,30 +34,38 @@ class BravoTVIE(AdobePassIE):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        settings = self._parse_json(self._search_regex(
-            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'),
+            r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>({.+?})</script>', webpage, 'drupal settings'),
            display_id)
        info = {}
        query = {
            'mbr': 'true',
        }
        account_pid, release_pid = [None] * 2
-        tve = settings.get('sharedTVE')
+        tve = settings.get('ls_tve')
        if tve:
            query['manifest'] = 'm3u'
-            account_pid = 'HNK2IC'
-            release_pid = tve['release_pid']
+            mobj = re.search(r'<[^>]+id="pdk-player"[^>]+data-url=["\']?(?:https?:)?//player\.theplatform\.com/p/([^/]+)/(?:[^/]+/)*select/([^?#&"\']+)', webpage)
+            if mobj:
+                account_pid, tp_path = mobj.groups()
+                release_pid = tp_path.strip('/').split('/')[-1]
+            else:
+                account_pid = 'HNK2IC'
+                tp_path = release_pid = tve['release_pid']
            if tve.get('entitlement') == 'auth':
-                adobe_pass = settings.get('adobePass', {})
+                adobe_pass = settings.get('tve_adobe_auth', {})
                resource = self._get_mvpd_resource(
                    adobe_pass.get('adobePassResourceId', 'bravo'),
                    tve['title'], release_pid, tve.get('rating'))
                query['auth'] = self._extract_mvpd_auth(
                    url, release_pid, adobe_pass.get('adobePassRequestorId', 'bravo'), resource)
        else:
-            shared_playlist = settings['shared_playlist']
+            shared_playlist = settings['ls_playlist']
            account_pid = shared_playlist['account_pid']
            metadata = shared_playlist['video_metadata'][shared_playlist['default_clip']]
-            release_pid = metadata['release_pid']
+            tp_path = release_pid = metadata.get('release_pid')
+            if not release_pid:
+                release_pid = metadata['guid']
+                tp_path = 'media/guid/2140479951/' + release_pid
            info.update({
                'title': metadata['title'],
                'description': metadata.get('description'),
@ -67,7 +77,7 @@ class BravoTVIE(AdobePassIE):
            '_type': 'url_transparent',
            'id': release_pid,
            'url': smuggle_url(update_url_query(
-                'http://link.theplatform.com/s/%s/%s' % (account_pid, release_pid),
+                'http://link.theplatform.com/s/%s/%s' % (account_pid, tp_path),
                query), {'force_smil_url': True}),
            'ie_key': 'ThePlatform',
        })
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@ -3,6 +3,7 @@ from __future__ import unicode_literals

 import json
 import os
+import random
 import re
 import subprocess
 import tempfile
@ -243,7 +244,7 @@ class PhantomJSwrapper(object):


 class OpenloadIE(InfoExtractor):
-    _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space))'
+    _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz|me)|openloed\.co)'
    _VALID_URL = r'''(?x)
                    https?://
                        (?P<host>
@ -350,9 +351,21 @@ class OpenloadIE(InfoExtractor):
    }, {
        'url': 'https://oload.space/f/IY4eZSst3u8/',
        'only_matching': True,
+    }, {
+        'url': 'https://oladblock.services/f/b8NWEgkqNLI/',
+        'only_matching': True,
+    }, {
+        'url': 'https://oladblock.xyz/f/b8NWEgkqNLI/',
+        'only_matching': True,
+    }, {
+        'url': 'https://oladblock.me/f/b8NWEgkqNLI/',
+        'only_matching': True,
+    }, {
+        'url': 'https://openloed.co/f/b8NWEgkqNLI/',
+        'only_matching': True,
    }]

-    _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
+    _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major}.0.{build}.{patch} Safari/537.36'

    @staticmethod
    def _extract_urls(webpage):
@ -367,7 +380,11 @@ class OpenloadIE(InfoExtractor):

        url_pattern = 'https://%s/%%s/%s/' % (host, video_id)
        headers = {
-            'User-Agent': self._USER_AGENT,
+            'User-Agent': self._USER_AGENT_TPL % {
+                'major': random.randint(63, 73),
+                'build': random.randint(3239, 3683),
+                'patch': random.randint(0, 100),
+            },
        }

        for path in ('embed', 'f'):
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@ -1,3 +1,3 @@
 from __future__ import unicode_literals

-__version__ = '2019.04.07'
+__version__ = '2019.04.17'