Merge branch 'master' of github.com:rg3/youtube-dl

* 'master' of github.com:rg3/youtube-dl:
  [vrt] fix extraction(closes #20527)
  [canvas] add support for vrtnieuws and sporza site ids and extract AES HLS formats
  [vrv] extract captions(closes #19238)

youtube_dl/extractor/canvas.py

@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrtvideo)/assets/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
         'md5': '90139b746a0a9bd7bb631283f6e2a64e',
@@ -35,6 +35,10 @@ class CanvasIE(InfoExtractor):
         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
         'only_matching': True,
     }]
+    _HLS_ENTRY_PROTOCOLS_MAP = {
+        'HLS': 'm3u8_native',
+        'HLS_AES': 'm3u8',
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -52,9 +56,9 @@ class CanvasIE(InfoExtractor):
             format_url, format_type = target.get('url'), target.get('type')
             if not format_url or not format_type:
                 continue
-            if format_type == 'HLS':
+            if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
                     m3u8_id=format_type, fatal=False))
             elif format_type == 'HDS':
                 formats.extend(self._extract_f4m_formats(

youtube_dl/extractor/vrt.py

@@ -5,150 +5,83 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    extract_attributes,
     float_or_none,
+    get_element_by_class,
+    strip_or_none,
+    unified_timestamp,
 )
 
 
 class VRTIE(InfoExtractor):
-    IE_DESC = 'deredactie.be, sporza.be, cobra.be and cobra.canvas.be'
+    IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza'
-    _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>vrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P<id>[^/?&#]+)'
-    _TESTS = [
+    _TESTS = [{
-        # deredactie.be
+        'url': 'https://www.vrt.be/vrtnws/nl/2019/05/15/beelden-van-binnenkant-notre-dame-een-maand-na-de-brand/',
-        {
+        'md5': 'e1663accf5cf13f375f3cd0d10476669',
-            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
-            'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
         'info_dict': {
-                'id': '2129880',
+            'id': 'pbs-pub-7855fc7b-1448-49bc-b073-316cb60caa71$vid-2ca50305-c38a-4762-9890-65cbd098b7bd',
-                'ext': 'flv',
-                'title': 'Het journaal L - 25/10/14',
-                'description': None,
-                'timestamp': 1414271750.949,
-                'upload_date': '20141025',
-                'duration': 929,
-            },
-            'skip': 'HTTP Error 404: Not Found',
-        },
-        # sporza.be
-        {
-            'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
-            'md5': '11f53088da9bf8e7cfc42456697953ff',
-            'info_dict': {
-                'id': '2124639',
-                'ext': 'flv',
-                'title': 'Bekijk Extra Time van 20 oktober',
-                'description': 'md5:83ac5415a4f1816c6a93f8138aef2426',
-                'timestamp': 1413835980.560,
-                'upload_date': '20141020',
-                'duration': 3238,
-            },
-            'skip': 'HTTP Error 404: Not Found',
-        },
-        # cobra.be
-        {
-            'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
-            'md5': '78a2b060a5083c4f055449a72477409d',
-            'info_dict': {
-                'id': '2126050',
-                'ext': 'flv',
-                'title': 'Bret Easton Ellis in Café Corsari',
-                'description': 'md5:f699986e823f32fd6036c1855a724ee9',
-                'timestamp': 1413967500.494,
-                'upload_date': '20141022',
-                'duration': 661,
-            },
-            'skip': 'HTTP Error 404: Not Found',
-        },
-        {
-            # YouTube video
-            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/nieuws/cultuurenmedia/1.2622957',
-            'md5': 'b8b93da1df1cea6c8556255a796b7d61',
-            'info_dict': {
-                'id': 'Wji-BZ0oCwg',
             'ext': 'mp4',
-                'title': 'ROGUE ONE: A STAR WARS STORY Official Teaser Trailer',
+            'title': 'Beelden van binnenkant Notre-Dame, één maand na de brand',
-                'description': 'md5:8e468944dce15567a786a67f74262583',
+            'description': 'Op maandagavond 15 april ging een deel van het dakgebinte van de Parijse kathedraal in vlammen op.',
-                'uploader': 'Star Wars',
+            'timestamp': 1557924660,
-                'uploader_id': 'starwars',
+            'upload_date': '20190515',
-                'upload_date': '20160407',
+            'duration': 31.2,
         },
-            'add_ie': ['Youtube'],
+    }, {
-        },
+        'url': 'https://sporza.be/nl/2019/05/15/de-belgian-cats-zijn-klaar-voor-het-ek/',
-        {
+        'md5': '910bba927566e9ab992278f647eb4b75',
-            'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
         'info_dict': {
-                'id': '2377055',
+            'id': 'pbs-pub-f2c86a46-8138-413a-a4b9-a0015a16ce2c$vid-1f112b31-e58e-4379-908d-aca6d80f8818',
             'ext': 'mp4',
-                'title': 'Cafe Derby',
+            'title': 'De Belgian Cats zijn klaar voor het EK mét Ann Wauters',
-                'description': 'Lenny Van Wesemael debuteert met de langspeelfilm Café Derby. Een waar gebeurd maar ook verzonnen verhaal.',
+            'timestamp': 1557923760,
-                'upload_date': '20150626',
+            'upload_date': '20190515',
-                'timestamp': 1435305240.769,
+            'duration': 115.17,
         },
-            'params': {
+    }, {
-                # m3u8 download
+        'url': 'https://www.vrt.be/vrtnws/en/2019/05/15/belgium_s-eurovision-entry-falls-at-the-first-hurdle/',
-                'skip_download': True,
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vrt.be/vrtnws/de/2019/05/15/aus-fuer-eliott-im-halbfinale-des-eurosongfestivals/',
+        'only_matching': True,
+    }]
+    _CLIENT_MAP = {
+        'vrt.be/vrtnws': 'vrtnieuws',
+        'sporza.be': 'sporza',
     }
-        }
-    ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        site, display_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, display_id)
+        attrs = extract_attributes(self._search_regex(
+            r'(<[^>]+class="vrtvideo"[^>]*>)', webpage, 'vrt video'))
 
-        webpage = self._download_webpage(url, video_id)
+        asset_id = attrs['data-videoid']
+        publication_id = attrs.get('data-publicationid')
+        if publication_id:
+            asset_id = publication_id + '$' + asset_id
+        client = attrs.get('data-client') or self._CLIENT_MAP[site]
 
-        video_id = self._search_regex(
+        title = strip_or_none(get_element_by_class(
-            r'data-video-id="([^"]+)_[^"]+"', webpage, 'video id', fatal=False)
+            'vrt-title', webpage) or self._html_search_meta(
-
+            ['og:title', 'twitter:title', 'name'], webpage))
-        src = self._search_regex(
+        description = self._html_search_meta(
-            r'data-video-src="([^"]+)"', webpage, 'video src', default=None)
+            ['og:description', 'twitter:description', 'description'], webpage)
-
+        if description == '…':
-        video_type = self._search_regex(
+            description = None
-            r'data-video-type="([^"]+)"', webpage, 'video type', default=None)
+        timestamp = unified_timestamp(self._html_search_meta(
-
+            'article:published_time', webpage))
-        if video_type == 'YouTubeVideo':
-            return self.url_result(src, 'Youtube')
-
-        formats = []
-
-        mobj = re.search(
-            r'data-video-iphone-server="(?P<server>[^"]+)"\s+data-video-iphone-path="(?P<path>[^"]+)"',
-            webpage)
-        if mobj:
-            formats.extend(self._extract_m3u8_formats(
-                '%s/%s' % (mobj.group('server'), mobj.group('path')),
-                video_id, 'mp4', m3u8_id='hls', fatal=False))
-
-        if src:
-            formats = self._extract_wowza_formats(src, video_id)
-            if 'data-video-geoblocking="true"' not in webpage:
-                for f in formats:
-                    if f['url'].startswith('rtsp://'):
-                        http_format = f.copy()
-                        http_format.update({
-                            'url': f['url'].replace('rtsp://', 'http://').replace('vod.', 'download.').replace('/_definst_/', '/').replace('mp4:', ''),
-                            'format_id': f['format_id'].replace('rtsp', 'http'),
-                            'protocol': 'http',
-                        })
-                        formats.append(http_format)
-
-        if not formats and 'data-video-geoblocking="true"' in webpage:
-            self.raise_geo_restricted('This video is only available in Belgium')
-
-        self._sort_formats(formats)
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage, default=None)
-        thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = float_or_none(self._search_regex(
-            r'data-video-sitestat-pubdate="(\d+)"', webpage, 'timestamp', fatal=False), 1000)
-        duration = float_or_none(self._search_regex(
-            r'data-video-duration="(\d+)"', webpage, 'duration', fatal=False), 1000)
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'id': asset_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnail': attrs.get('data-posterimage'),
             'timestamp': timestamp,
-            'duration': duration,
+            'duration': float_or_none(attrs.get('data-duration'), 1000),
-            'formats': formats,
+            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (client, asset_id),
+            'ie_key': 'Canvas',
         }

youtube_dl/extractor/vrv.py

@@ -198,7 +198,8 @@ class VRVIE(VRVBaseIE):
         self._sort_formats(formats)
 
         subtitles = {}
-        for subtitle in streams_json.get('subtitles', {}).values():
+        for k in ('captions', 'subtitles'):
+            for subtitle in streams_json.get(k, {}).values():
                 subtitle_url = subtitle.get('url')
                 if not subtitle_url:
                     continue