From d37dc6e1c956ce0f390a4b832f8388cb94254f90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Aug 2018 23:27:08 +0700
Subject: [PATCH 001/510] [clyp] Add support for token protected media (closes
 #17184)

---
 youtube_dl/extractor/clyp.py | 35 ++++++++++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
index 57e643799..06d04de13 100644
--- a/youtube_dl/extractor/clyp.py
+++ b/youtube_dl/extractor/clyp.py
@@ -1,15 +1,19 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     float_or_none,
-    parse_iso8601,
+    unified_timestamp,
 )
 
 
 class ClypIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://clyp.it/ojz2wfah',
         'md5': '1d4961036c41247ecfdcc439c0cddcbb',
         'info_dict': {
@@ -21,13 +25,34 @@ class ClypIE(InfoExtractor):
             'timestamp': 1443515251,
             'upload_date': '20150929',
         },
-    }
+    }, {
+        'url': 'https://clyp.it/b04p1odi?token=b0078e077e15835845c528a44417719d',
+        'info_dict': {
+            'id': 'b04p1odi',
+            'ext': 'mp3',
+            'title': 'GJ! (Reward Edit)',
+            'description': 'Metal Resistance (THE ONE edition)',
+            'duration': 177.789,
+            'timestamp': 1528241278,
+            'upload_date': '20180605',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         audio_id = self._match_id(url)
 
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        token = qs.get('token', [None])[0]
+
+        query = {}
+        if token:
+            query['token'] = token
+
         metadata = self._download_json(
-            'https://api.clyp.it/%s' % audio_id, audio_id)
+            'https://api.clyp.it/%s' % audio_id, audio_id, query=query)
 
         formats = []
         for secure in ('', 'Secure'):
@@ -45,7 +70,7 @@ class ClypIE(InfoExtractor):
         title = metadata['Title']
         description = metadata.get('Description')
         duration = float_or_none(metadata.get('Duration'))
-        timestamp = parse_iso8601(metadata.get('DateCreated'))
+        timestamp = unified_timestamp(metadata.get('DateCreated'))
 
         return {
             'id': audio_id,

From b65e3b0636c4b992f3187bff7d5268c6891023a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Aug 2018 01:47:10 +0700
Subject: [PATCH 002/510] [bitchute] Add extractor (closes #14052)

---
 youtube_dl/extractor/bitchute.py   | 116 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   4 +
 2 files changed, 120 insertions(+)
 create mode 100644 youtube_dl/extractor/bitchute.py

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
new file mode 100644
index 000000000..28016f18b
--- /dev/null
+++ b/youtube_dl/extractor/bitchute.py
@@ -0,0 +1,116 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+import re
+
+from .common import InfoExtractor
+from ..utils import urlencode_postdata
+
+
+class BitChuteIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bitchute\.com/(?:video|embed|torrent/[^/]+)/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.bitchute.com/video/szoMrox2JEI/',
+        'md5': '66c4a70e6bfc40dcb6be3eb1d74939eb',
+        'info_dict': {
+            'id': 'szoMrox2JEI',
+            'ext': 'mp4',
+            'title': 'Fuck bitches get money',
+            'description': 'md5:3f21f6fb5b1d17c3dee9cf6b5fe60b3a',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Victoria X Rave',
+        },
+    }, {
+        'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.bitchute.com/torrent/Zee5BE49045h/szoMrox2JEI.webtorrent',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://www.bitchute.com/video/%s' % video_id, video_id)
+
+        title = self._search_regex(
+            (r'<[^>]+\bid=["\']video-title[^>]+>([^<]+)', r'<title>([^<]+)'),
+            webpage, 'title', default=None) or self._html_search_meta(
+            'description', webpage, 'title',
+            default=None) or self._og_search_description(webpage)
+
+        formats = [
+            {'url': mobj.group('url')}
+            for mobj in re.finditer(
+                r'addWebSeed\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage)]
+        self._sort_formats(formats)
+
+        description = self._html_search_regex(
+            r'(?s)<div\b[^>]+\bclass=["\']full hidden[^>]+>(.+?)</div>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(
+            webpage, default=None) or self._html_search_meta(
+            'twitter:image:src', webpage, 'thumbnail')
+        uploader = self._html_search_regex(
+            r'(?s)<p\b[^>]+\bclass=["\']video-author[^>]+>(.+?)</p>', webpage,
+            'uploader', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'formats': formats,
+        }
+
+
+class BitChuteChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bitchute\.com/channel/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://www.bitchute.com/channel/victoriaxrave/',
+        'playlist_mincount': 185,
+        'info_dict': {
+            'id': 'victoriaxrave',
+        },
+    }
+
+    _TOKEN = 'zyG6tQcGPE5swyAEFLqKUwMuMMuF6IO2DZ6ZDQjGfsL0e4dcTLwqkTTul05Jdve7'
+
+    def _entries(self, channel_id):
+        channel_url = 'https://www.bitchute.com/channel/%s/' % channel_id
+        for page_num in itertools.count(0):
+            data = self._download_json(
+                '%sextend/' % channel_url, channel_id,
+                'Downloading channel page %d' % (page_num + 1),
+                data=urlencode_postdata({
+                    'csrfmiddlewaretoken': self._TOKEN,
+                    'name': '',
+                    'offset': page_num * 25
+                }), headers={
+                    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                    'Referer': channel_url,
+                    'X-Requested-With': 'XMLHttpRequest',
+                    'Cookie': 'csrftoken=%s' % self._TOKEN,
+                })
+            if data.get('success') is False:
+                break
+            html = data.get('html')
+            if not html:
+                break
+            video_ids = re.findall(
+                r'class=["\']channel-videos-image-container[^>]+>\s*<a\b[^>]+\bhref=["\']/video/([^"\'/]+)',
+                html)
+            if not video_ids:
+                break
+            for video_id in video_ids:
+                yield self.url_result(
+                    'https://www.bitchute.com/video/%s' % video_id,
+                    ie=BitChuteIE.ie_key(), video_id=video_id)
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        return self.playlist_result(
+            self._entries(channel_id), playlist_id=channel_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c7a91a986..eb22d0999 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -118,6 +118,10 @@ from .bilibili import (
     BiliBiliBangumiIE,
 )
 from .biobiochiletv import BioBioChileTVIE
+from .bitchute import (
+    BitChuteIE,
+    BitChuteChannelIE,
+)
 from .biqle import BIQLEIE
 from .bleacherreport import (
     BleacherReportIE,

From de4c41b437c7a0126074d75ddeeccbf4470d0684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Aug 2018 01:52:50 +0700
Subject: [PATCH 003/510] [bitchute] Improve page offset

---
 youtube_dl/extractor/bitchute.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 28016f18b..da263714a 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -81,14 +81,15 @@ class BitChuteChannelIE(InfoExtractor):
 
     def _entries(self, channel_id):
         channel_url = 'https://www.bitchute.com/channel/%s/' % channel_id
-        for page_num in itertools.count(0):
+        offset = 0
+        for page_num in itertools.count(1):
             data = self._download_json(
                 '%sextend/' % channel_url, channel_id,
-                'Downloading channel page %d' % (page_num + 1),
+                'Downloading channel page %d' % page_num,
                 data=urlencode_postdata({
                     'csrfmiddlewaretoken': self._TOKEN,
                     'name': '',
-                    'offset': page_num * 25
+                    'offset': offset,
                 }), headers={
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                     'Referer': channel_url,
@@ -105,6 +106,7 @@ class BitChuteChannelIE(InfoExtractor):
                 html)
             if not video_ids:
                 break
+            offset += len(video_ids)
             for video_id in video_ids:
                 yield self.url_result(
                     'https://www.bitchute.com/video/%s' % video_id,

From 4779420ce8e13d0df33189907e477146a2a0cefe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 12 Aug 2018 05:31:18 +0100
Subject: [PATCH 004/510] [redbulltv] add support redbull.com tv URLs(closes
 #17218)

---
 youtube_dl/extractor/redbulltv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index 243603676..7e8d58f38 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class RedBullTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/video/(?P<id>AP-\w+)'
+    _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com/(?:[^/]+/)?tv)/video/(?P<id>AP-\w+)'
     _TESTS = [{
         # film
         'url': 'https://www.redbull.tv/video/AP-1Q6XCDTAN1W11',
@@ -35,6 +35,9 @@ class RedBullTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.redbull.com/int-en/tv/video/AP-1UWHCAR9S1W11/rob-meets-sam-gaze?playlist=playlists::3f81040a-2f31-4832-8e2e-545b1d39d173',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 24e0cd709fae13d9192361dae27ecd292c6b1fd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Aug 2018 00:15:59 +0700
Subject: [PATCH 005/510] [raywenderlich] Adapt to site redesign (closes
 #17225)

---
 youtube_dl/extractor/extractors.py    |   5 +-
 youtube_dl/extractor/raywenderlich.py | 189 ++++++++++++++++++--------
 2 files changed, 137 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index eb22d0999..23ad42da9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -899,7 +899,10 @@ from .rai import (
     RaiPlayPlaylistIE,
     RaiIE,
 )
-from .raywenderlich import RayWenderlichIE
+from .raywenderlich import (
+    RayWenderlichIE,
+    RayWenderlichCourseIE,
+)
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redbulltv import RedBullTVIE
diff --git a/youtube_dl/extractor/raywenderlich.py b/youtube_dl/extractor/raywenderlich.py
index 640c3ee23..5411ece21 100644
--- a/youtube_dl/extractor/raywenderlich.py
+++ b/youtube_dl/extractor/raywenderlich.py
@@ -4,24 +4,37 @@ import re
 
 from .common import InfoExtractor
 from .vimeo import VimeoIE
+from ..compat import compat_str
 from ..utils import (
-    extract_attributes,
     ExtractorError,
-    smuggle_url,
-    unsmuggle_url,
+    int_or_none,
+    merge_dicts,
+    try_get,
+    unescapeHTML,
+    unified_timestamp,
     urljoin,
 )
 
 
 class RayWenderlichIE(InfoExtractor):
-    _VALID_URL = r'https?://videos\.raywenderlich\.com/courses/(?P<course_id>[^/]+)/lessons/(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            videos\.raywenderlich\.com/courses|
+                            (?:www\.)?raywenderlich\.com
+                        )/
+                        (?P<course_id>[^/]+)/lessons/(?P<id>\d+)
+                    '''
 
     _TESTS = [{
-        'url': 'https://videos.raywenderlich.com/courses/105-testing-in-ios/lessons/1',
+        'url': 'https://www.raywenderlich.com/3530-testing-in-ios/lessons/1',
         'info_dict': {
             'id': '248377018',
             'ext': 'mp4',
-            'title': 'Testing In iOS Episode 1: Introduction',
+            'title': 'Introduction',
+            'description': 'md5:804d031b3efa9fcb49777d512d74f722',
+            'timestamp': 1513906277,
+            'upload_date': '20171222',
             'duration': 133,
             'uploader': 'Ray Wenderlich',
             'uploader_id': 'user3304672',
@@ -34,69 +47,133 @@ class RayWenderlichIE(InfoExtractor):
         'expected_warnings': ['HTTP Error 403: Forbidden'],
     }, {
         'url': 'https://videos.raywenderlich.com/courses/105-testing-in-ios/lessons/1',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_video_id(data, lesson_id):
+        if not data:
+            return
+        groups = try_get(data, lambda x: x['groups'], list) or []
+        if not groups:
+            return
+        for group in groups:
+            if not isinstance(group, dict):
+                continue
+            contents = try_get(data, lambda x: x['contents'], list) or []
+            for content in contents:
+                if not isinstance(content, dict):
+                    continue
+                ordinal = int_or_none(content.get('ordinal'))
+                if ordinal != lesson_id:
+                    continue
+                video_id = content.get('identifier')
+                if video_id:
+                    return compat_str(video_id)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_id, lesson_id = mobj.group('course_id', 'id')
+        display_id = '%s/%s' % (course_id, lesson_id)
+
+        webpage = self._download_webpage(url, display_id)
+
+        thumbnail = self._og_search_thumbnail(
+            webpage, default=None) or self._html_search_meta(
+            'twitter:image', webpage, 'thumbnail')
+
+        if '>Subscribe to unlock' in webpage:
+            raise ExtractorError(
+                'This content is only available for subscribers',
+                expected=True)
+
+        info = {
+            'thumbnail': thumbnail,
+        }
+
+        vimeo_id = self._search_regex(
+            r'data-vimeo-id=["\'](\d+)', webpage, 'vimeo id', default=None)
+
+        if not vimeo_id:
+            data = self._parse_json(
+                self._search_regex(
+                    r'data-collection=(["\'])(?P<data>{.+?})\1', webpage,
+                    'data collection', default='{}', group='data'),
+                display_id, transform_source=unescapeHTML, fatal=False)
+            video_id = self._extract_video_id(
+                data, lesson_id) or self._search_regex(
+                r'/videos/(\d+)/', thumbnail, 'video id')
+            headers = {
+                'Referer': url,
+                'X-Requested-With': 'XMLHttpRequest',
+            }
+            csrf_token = self._html_search_meta(
+                'csrf-token', webpage, 'csrf token', default=None)
+            if csrf_token:
+                headers['X-CSRF-Token'] = csrf_token
+            video = self._download_json(
+                'https://videos.raywenderlich.com/api/v1/videos/%s.json'
+                % video_id, display_id, headers=headers)['video']
+            vimeo_id = video['clips'][0]['provider_id']
+            info.update({
+                '_type': 'url_transparent',
+                'title': video.get('name'),
+                'description': video.get('description') or video.get(
+                    'meta_description'),
+                'duration': int_or_none(video.get('duration')),
+                'timestamp': unified_timestamp(video.get('created_at')),
+            })
+
+        return merge_dicts(info, self.url_result(
+            VimeoIE._smuggle_referrer(
+                'https://player.vimeo.com/video/%s' % vimeo_id, url),
+            ie=VimeoIE.ie_key(), video_id=vimeo_id))
+
+
+class RayWenderlichCourseIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            videos\.raywenderlich\.com/courses|
+                            (?:www\.)?raywenderlich\.com
+                        )/
+                        (?P<id>[^/]+)
+                    '''
+
+    _TEST = {
+        'url': 'https://www.raywenderlich.com/3530-testing-in-ios',
         'info_dict': {
             'title': 'Testing in iOS',
-            'id': '105-testing-in-ios',
+            'id': '3530-testing-in-ios',
         },
         'params': {
             'noplaylist': False,
         },
         'playlist_count': 29,
-    }]
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if RayWenderlichIE.suitable(url) else super(
+            RayWenderlichCourseIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        url, smuggled_data = unsmuggle_url(url, {})
+        course_id = self._match_id(url)
 
-        mobj = re.match(self._VALID_URL, url)
-        course_id, lesson_id = mobj.group('course_id', 'id')
-        video_id = '%s/%s' % (course_id, lesson_id)
-
-        webpage = self._download_webpage(url, video_id)
-
-        no_playlist = self._downloader.params.get('noplaylist')
-        if no_playlist or smuggled_data.get('force_video', False):
-            if no_playlist:
-                self.to_screen(
-                    'Downloading just video %s because of --no-playlist'
-                    % video_id)
-            if '>Subscribe to unlock' in webpage:
-                raise ExtractorError(
-                    'This content is only available for subscribers',
-                    expected=True)
-            vimeo_id = self._search_regex(
-                r'data-vimeo-id=["\'](\d+)', webpage, 'video id')
-            return self.url_result(
-                VimeoIE._smuggle_referrer(
-                    'https://player.vimeo.com/video/%s' % vimeo_id, url),
-                ie=VimeoIE.ie_key(), video_id=vimeo_id)
-
-        self.to_screen(
-            'Downloading playlist %s - add --no-playlist to just download video'
-            % course_id)
-
-        lesson_ids = set((lesson_id, ))
-        for lesson in re.findall(
-                r'(<a[^>]+\bclass=["\']lesson-link[^>]+>)', webpage):
-            attrs = extract_attributes(lesson)
-            if not attrs:
-                continue
-            lesson_url = attrs.get('href')
-            if not lesson_url:
-                continue
-            lesson_id = self._search_regex(
-                r'/lessons/(\d+)', lesson_url, 'lesson id', default=None)
-            if not lesson_id:
-                continue
-            lesson_ids.add(lesson_id)
+        webpage = self._download_webpage(url, course_id)
 
         entries = []
-        for lesson_id in sorted(lesson_ids):
+        lesson_urls = set()
+        for lesson_url in re.findall(
+                r'<a[^>]+\bhref=["\'](/%s/lessons/\d+)' % course_id, webpage):
+            if lesson_url in lesson_urls:
+                continue
+            lesson_urls.add(lesson_url)
             entries.append(self.url_result(
-                smuggle_url(urljoin(url, lesson_id), {'force_video': True}),
-                ie=RayWenderlichIE.ie_key()))
+                urljoin(url, lesson_url), ie=RayWenderlichIE.ie_key()))
 
-        title = self._search_regex(
-            r'class=["\']course-title[^>]+>([^<]+)', webpage, 'course title',
-            default=None)
+        title = self._og_search_title(
+            webpage, default=None) or self._html_search_meta(
+            'twitter:title', webpage, 'title', default=None)
 
         return self.playlist_result(entries, course_id, title)

From 57c68ec4c303f471b2519db21b9b56c080d88338 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Aug 2018 22:51:44 +0700
Subject: [PATCH 006/510] [generic] Add support for expressen embeds

---
 youtube_dl/extractor/expressen.py | 23 ++++++++++++++++++++++-
 youtube_dl/extractor/generic.py   |  6 ++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/expressen.py b/youtube_dl/extractor/expressen.py
index f61178012..934571472 100644
--- a/youtube_dl/extractor/expressen.py
+++ b/youtube_dl/extractor/expressen.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
@@ -11,7 +13,13 @@ from ..utils import (
 
 
 class ExpressenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?expressen\.se/tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?expressen\.se/
+                        (?:(?:tvspelare/video|videoplayer/embed)/)?
+                        tv/(?:[^/]+/)*
+                        (?P<id>[^/?#&]+)
+                    '''
     _TESTS = [{
         'url': 'https://www.expressen.se/tv/ledare/ledarsnack/ledarsnack-om-arbetslosheten-bland-kvinnor-i-speciellt-utsatta-omraden/',
         'md5': '2fbbe3ca14392a6b1b36941858d33a45',
@@ -28,8 +36,21 @@ class ExpressenIE(InfoExtractor):
     }, {
         'url': 'https://www.expressen.se/tv/kultur/kulturdebatt-med-expressens-karin-olsson/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.expressen.se/tvspelare/video/tv/ditv/ekonomistudion/experterna-har-ar-fragorna-som-avgor-valet/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.expressen.se/videoplayer/embed/tv/ditv/ekonomistudion/experterna-har-ar-fragorna-som-avgor-valet/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
+        'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [
+            mobj.group('url') for mobj in re.finditer(
+                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?expressen\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1',
+                webpage)]
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 43218c3a4..83a31f3d3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -114,6 +114,7 @@ from .indavideo import IndavideoEmbedIE
 from .apa import APAIE
 from .foxnews import FoxNewsIE
 from .viqeo import ViqeoIE
+from .expressen import ExpressenIE
 
 
 class GenericIE(InfoExtractor):
@@ -3109,6 +3110,11 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 viqeo_urls, video_id, video_title, ie=ViqeoIE.ie_key())
 
+        expressen_urls = ExpressenIE._extract_urls(webpage)
+        if expressen_urls:
+            return self.playlist_from_matches(
+                expressen_urls, video_id, video_title, ie=ExpressenIE.ie_key())
+
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:

From 60c0856223391864135694197db2d17125fe4e8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Aug 2018 23:27:12 +0700
Subject: [PATCH 007/510] [utils] Use pure browser header for User-Agent
 (closes #17236)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 29cafd8f0..0c830ba71 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -82,7 +82,7 @@ def register_socks_protocols():
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From d6ef8b4dd4230233b8ada6b1f8d7083df86a7cd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Aug 2018 00:11:41 +0700
Subject: [PATCH 008/510] [nova] Fix extraction (closes #17241)

---
 youtube_dl/extractor/nova.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 06cb8cb3f..edf8b82c6 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -6,7 +6,9 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    js_to_json,
     unified_strdate,
+    url_or_none,
 )
 
 
@@ -111,8 +113,21 @@ class NovaIE(InfoExtractor):
             webpage, 'video id')
 
         config_url = self._search_regex(
-            r'src="(http://tn\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
+            r'src="(https?://(?:tn|api)\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
             webpage, 'config url', default=None)
+        config_params = {}
+
+        if not config_url:
+            player = self._parse_json(
+                self._search_regex(
+                    r'(?s)Player\s*\(.+?\s*,\s*({.+?\bmedia\b["\']?\s*:\s*["\']?\d+.+?})\s*\)', webpage,
+                    'player', default='{}'),
+                video_id, transform_source=js_to_json, fatal=False)
+            if player:
+                config_url = url_or_none(player.get('configUrl'))
+                params = player.get('configParams')
+                if isinstance(params, dict):
+                    config_params = params
 
         if not config_url:
             DEFAULT_SITE_ID = '23000'
@@ -127,14 +142,20 @@ class NovaIE(InfoExtractor):
             }
 
             site_id = self._search_regex(
-                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(site, DEFAULT_SITE_ID)
+                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(
+                site, DEFAULT_SITE_ID)
 
-            config_url = ('http://tn.nova.cz/bin/player/videojs/config.php?site=%s&media=%s&jsVar=vjsconfig'
-                          % (site_id, video_id))
+            config_url = 'https://api.nova.cz/bin/player/videojs/config.php'
+            config_params = {
+                'site': site_id,
+                'media': video_id,
+                'quality': 3,
+                'version': 1,
+            }
 
         config = self._download_json(
             config_url, display_id,
-            'Downloading config JSON',
+            'Downloading config JSON', query=config_params,
             transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
 
         mediafile = config['mediafile']

From 0a74b4519147aec45dc39756dc55b5755beefbc2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 17 Aug 2018 11:59:22 +0100
Subject: [PATCH 009/510] [cwtv] fix extraction(closes #17256)

---
 youtube_dl/extractor/cwtv.py | 82 +++++++++++++-----------------------
 1 file changed, 29 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index f4cf0f1c5..224a1fb5d 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    parse_age_limit,
     parse_iso8601,
+    smuggle_url,
+    str_or_none,
 )
 
 
@@ -40,10 +43,15 @@ class CWTVIE(InfoExtractor):
             'duration': 1263,
             'series': 'Whose Line Is It Anyway?',
             'season_number': 11,
-            'season': '11',
             'episode_number': 20,
             'upload_date': '20151006',
             'timestamp': 1444107300,
+            'age_limit': 14,
+            'uploader': 'CWTV',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
     }, {
         'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
@@ -58,60 +66,28 @@ class CWTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = None
-        formats = []
-        for partner in (154, 213):
-            vdata = self._download_json(
-                'http://metaframe.digitalsmiths.tv/v2/CWtv/assets/%s/partner/%d?format=json' % (video_id, partner), video_id, fatal=False)
-            if not vdata:
-                continue
-            video_data = vdata
-            for quality, quality_data in vdata.get('videos', {}).items():
-                quality_url = quality_data.get('uri')
-                if not quality_url:
-                    continue
-                if quality == 'variantplaylist':
-                    formats.extend(self._extract_m3u8_formats(
-                        quality_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
-                else:
-                    tbr = int_or_none(quality_data.get('bitrate'))
-                    format_id = 'http' + ('-%d' % tbr if tbr else '')
-                    if self._is_valid_url(quality_url, video_id, format_id):
-                        formats.append({
-                            'format_id': format_id,
-                            'url': quality_url,
-                            'tbr': tbr,
-                        })
-        video_metadata = video_data['assetFields']
-        ism_url = video_metadata.get('smoothStreamingUrl')
-        if ism_url:
-            formats.extend(self._extract_ism_formats(
-                ism_url, video_id, ism_id='mss', fatal=False))
-        self._sort_formats(formats)
+        video_data = self._download_json(
+            'http://images.cwtv.com/feed/mobileapp/video-meta/apiversion_8/guid_' + video_id,
+            video_id)['video']
+        title = video_data['title']
+        mpx_url = video_data.get('mpx_url') or 'http://link.theplatform.com/s/cwtv/media/guid/2703454149/%s?formats=M3U' % video_id
 
-        thumbnails = [{
-            'url': image['uri'],
-            'width': image.get('width'),
-            'height': image.get('height'),
-        } for image_id, image in video_data['images'].items() if image.get('uri')] if video_data.get('images') else None
-
-        subtitles = {
-            'en': [{
-                'url': video_metadata['UnicornCcUrl'],
-            }],
-        } if video_metadata.get('UnicornCcUrl') else None
+        season = str_or_none(video_data.get('season'))
+        episode = str_or_none(video_data.get('episode'))
+        if episode and season:
+            episode = episode.lstrip(season)
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
-            'title': video_metadata['title'],
-            'description': video_metadata.get('description'),
-            'duration': int_or_none(video_metadata.get('duration')),
-            'series': video_metadata.get('seriesName'),
-            'season_number': int_or_none(video_metadata.get('seasonNumber')),
-            'season': video_metadata.get('seasonName'),
-            'episode_number': int_or_none(video_metadata.get('episodeNumber')),
-            'timestamp': parse_iso8601(video_data.get('startTime')),
-            'thumbnails': thumbnails,
-            'formats': formats,
-            'subtitles': subtitles,
+            'title': title,
+            'url': smuggle_url(mpx_url, {'force_smil_url': True}),
+            'description': video_data.get('description_long'),
+            'duration': int_or_none(video_data.get('duration_secs')),
+            'series': video_data.get('series_name'),
+            'season_number': int_or_none(season),
+            'episode_number': int_or_none(episode),
+            'timestamp': parse_iso8601(video_data.get('start_time')),
+            'age_limit': parse_age_limit(video_data.get('rating')),
+            'ie_key': 'ThePlatform',
         }

From 6f356cbbcf0bb3e3e3abce912d451d1852999b67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Aug 2018 02:05:07 +0700
Subject: [PATCH 010/510] [bbccouk] Extend _ID_REGEX (closes #17270)

---
 youtube_dl/extractor/bbc.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 641bf6073..abcfa301d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -29,7 +29,7 @@ from ..compat import (
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _ID_REGEX = r'[pbw][\da-z]{7}'
+    _ID_REGEX = r'(?:[pbm][\da-z]{7}|w[\da-z]{7,14})'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?bbc\.co\.uk/
@@ -236,6 +236,12 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/programmes/w3csv1y9',
             'only_matching': True,
+        }, {
+            'url': 'https://www.bbc.co.uk/programmes/m00005xn',
+            'only_matching': True,
+        }, {
+            'url': 'https://www.bbc.co.uk/programmes/w172w4dww1jqt5s',
+            'only_matching': True,
         }]
 
     _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'

From bf1245d236f4199497a8cbc42ff0baa5f92f9edc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Aug 2018 02:15:48 +0700
Subject: [PATCH 011/510] [lci] Fix extraction (closes #17274)

---
 youtube_dl/extractor/lci.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lci.py b/youtube_dl/extractor/lci.py
index af34829e7..920872f5c 100644
--- a/youtube_dl/extractor/lci.py
+++ b/youtube_dl/extractor/lci.py
@@ -20,5 +20,7 @@ class LCIIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        wat_id = self._search_regex(r'data-watid=[\'"](\d+)', webpage, 'wat id')
+        wat_id = self._search_regex(
+            (r'data-watid=[\'"](\d+)', r'idwat["\']?\s*:\s*["\']?(\d+)'),
+            webpage, 'wat id')
         return self.url_result('wat:' + wat_id, 'Wat', wat_id)

From eda86b4335a464c2744adb537f1296eff24f5bde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Aug 2018 23:45:18 +0700
Subject: [PATCH 012/510] [anvato] Fallback to generic API key for access key
 to API key lookup (closes #16788, closes #17254)

---
 youtube_dl/extractor/anvato.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index f6a78eb5d..84e841035 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -134,9 +134,33 @@ class AnvatoIE(InfoExtractor):
         'telemundo': 'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582'
     }
 
+    _API_KEY = '3hwbSuqqT690uxjNYBktSQpa5ZrpYYR0Iofx7NcJHyA'
+
     _ANVP_RE = r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1'
     _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'
 
+    _TESTS = [{
+        # from https://www.boston25news.com/news/watch-humpback-whale-breaches-right-next-to-fishing-boat-near-nh/817484874
+        'url': 'anvato:8v9BEynrwx8EFLYpgfOWcG1qJqyXKlRM:4465496',
+        'info_dict': {
+            'id': '4465496',
+            'ext': 'mp4',
+            'title': 'VIDEO: Humpback whale breaches right next to NH boat',
+            'description': 'VIDEO: Humpback whale breaches right next to NH boat. Footage courtesy: Zach Fahey.',
+            'duration': 22,
+            'timestamp': 1534855680,
+            'upload_date': '20180821',
+            'uploader': 'ANV',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # from https://sanfrancisco.cbslocal.com/2016/06/17/source-oakland-cop-on-leave-for-having-girlfriend-help-with-police-reports/
+        'url': 'anvato:DVzl9QRzox3ZZsP9bNu5Li3X7obQOnqP:3417601',
+        'only_matching': True,
+    }]
+
     def __init__(self, *args, **kwargs):
         super(AnvatoIE, self).__init__(*args, **kwargs)
         self.__server_time = None
@@ -169,7 +193,8 @@ class AnvatoIE(InfoExtractor):
             'api': {
                 'anvrid': anvrid,
                 'anvstk': md5_text('%s|%s|%d|%s' % (
-                    access_key, anvrid, server_time, self._ANVACK_TABLE[access_key])),
+                    access_key, anvrid, server_time,
+                    self._ANVACK_TABLE.get(access_key, self._API_KEY))),
                 'anvts': server_time,
             },
         }
@@ -284,5 +309,6 @@ class AnvatoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         access_key, video_id = mobj.group('access_key_or_mcp', 'id')
         if access_key not in self._ANVACK_TABLE:
-            access_key = self._MCP_TO_ACCESS_KEY_TABLE[access_key]
+            access_key = self._MCP_TO_ACCESS_KEY_TABLE.get(
+                access_key) or access_key
         return self._get_anvato_videos(access_key, video_id)

From 28f96cf407cecbac21fe2c7d3417928dae3ab3b5 Mon Sep 17 00:00:00 2001
From: hmlinaric <mlinaric.hrvoje@gmail.com>
Date: Tue, 21 Aug 2018 19:06:27 +0200
Subject: [PATCH 013/510] [6play] Add support for play.rtl.hr

---
 youtube_dl/extractor/sixplay.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index a363221bc..207ab4477 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 class SixPlayIE(InfoExtractor):
     IE_NAME = '6play'
-    _VALID_URL = r'(?:6play:|https?://(?:www\.)?(?P<domain>6play\.fr|rtlplay.be)/.+?-c_)(?P<id>[0-9]+)'
+    _VALID_URL = r'(?:6play:|https?://(?:www\.)?(?P<domain>6play\.fr|rtlplay\.be|play\.rtl\.hr)/.+?-c_)(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://www.6play.fr/minute-par-minute-p_9533/le-but-qui-a-marque-lhistoire-du-football-francais-c_12041051',
         'md5': '31fcd112637baa0c2ab92c4fcd8baf27',
@@ -32,6 +32,9 @@ class SixPlayIE(InfoExtractor):
     }, {
         'url': 'https://www.rtlplay.be/rtl-info-13h-p_8551/les-titres-du-rtlinfo-13h-c_12045869',
         'only_matching': True,
+    }, {
+        'url': 'https://play.rtl.hr/pj-masks-p_9455/epizoda-34-sezona-1-catboyevo-cudo-na-dva-kotaca-c_11984989',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -39,6 +42,7 @@ class SixPlayIE(InfoExtractor):
         service, consumer_name = {
             '6play.fr': ('6play', 'm6web'),
             'rtlplay.be': ('rtlbe_rtl_play', 'rtlbe'),
+            'play.rtl.hr': ('rtlhr_rtl_play', 'rtlhr'),
         }.get(domain, ('6play', 'm6web'))
 
         data = self._download_json(

From 52007de8ca6fcc2b72cbfb3fe026ea63cda88f25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Aug 2018 01:14:47 +0700
Subject: [PATCH 014/510] [go] Add support for disneynow.go.com (closes #16299,
 closes #17264)

---
 youtube_dl/extractor/go.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index e781405f2..ec9dd6e3a 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -36,7 +36,8 @@ class GoIE(AdobePassIE):
             'requestor_id': 'DisneyXD',
         }
     }
-    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:(?:[^/]+/)*(?P<id>vdka\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:(?:[^/]+/)*(?P<id>vdka\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))'\
+                 % '|'.join(list(_SITE_INFO.keys()) + ['disneynow'])
     _TESTS = [{
         'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
         'info_dict': {
@@ -62,6 +63,14 @@ class GoIE(AdobePassIE):
     }, {
         'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
         'only_matching': True,
+    }, {
+        # brand 004
+        'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
+        'only_matching': True,
+    }, {
+        # brand 008
+        'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
+        'only_matching': True,
     }]
 
     def _extract_videos(self, brand, video_id='-1', show_id='-1'):
@@ -72,14 +81,23 @@ class GoIE(AdobePassIE):
 
     def _real_extract(self, url):
         sub_domain, video_id, display_id = re.match(self._VALID_URL, url).groups()
-        site_info = self._SITE_INFO[sub_domain]
-        brand = site_info['brand']
-        if not video_id:
-            webpage = self._download_webpage(url, display_id)
+        site_info = self._SITE_INFO.get(sub_domain, {})
+        brand = site_info.get('brand')
+        if not video_id or not site_info:
+            webpage = self._download_webpage(url, display_id or video_id)
             video_id = self._search_regex(
                 # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
                 # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
-                r'data-video-id=["\']*(VDKA\w+)', webpage, 'video id', default=None)
+                r'data-video-id=["\']*(VDKA\w+)', webpage, 'video id',
+                default=None)
+            if not site_info:
+                brand = self._search_regex(
+                    (r'data-brand=\s*["\']\s*(\d+)',
+                     r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
+                    default='004')
+                site_info = next(
+                    si for _, si in self._SITE_INFO.items()
+                    if si.get('brand') == brand)
             if not video_id:
                 # show extraction works for Disney, DisneyJunior and DisneyXD
                 # ABC and Freeform has different layout

From db192b29329696547b422d904819321077efddfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Aug 2018 01:44:22 +0700
Subject: [PATCH 015/510] [yourporn] Add extractor (closes #17298)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/yourporn.py   | 41 ++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/yourporn.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 23ad42da9..d8e86c277 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1424,6 +1424,7 @@ from .younow import (
     YouNowMomentIE,
 )
 from .youporn import YouPornIE
+from .yourporn import YourPornIE
 from .yourupload import YourUploadIE
 from .youtube import (
     YoutubeIE,
diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py
new file mode 100644
index 000000000..6602f7c03
--- /dev/null
+++ b/youtube_dl/extractor/yourporn.py
@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import urljoin
+
+
+class YourPornIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?yourporn\.sexy/post/(?P<id>[^/?#&.]+)'
+    _TEST = {
+        'url': 'https://yourporn.sexy/post/57ffcb2e1179b.html',
+        'md5': '6f8682b6464033d87acaa7a8ff0c092e',
+        'info_dict': {
+            'id': '57ffcb2e1179b',
+            'ext': 'mp4',
+            'title': 'md5:c9f43630bd968267672651ba905a7d35',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = urljoin(url, self._parse_json(
+            self._search_regex(
+                r'data-vnfo=(["\'])(?P<data>{.+?})\1', webpage, 'data info',
+                group='data'),
+            video_id)[video_id])
+
+        title = (self._search_regex(
+            r'<[^>]+\bclass=["\']PostEditTA[^>]+>([^<]+)', webpage, 'title',
+            default=None) or self._og_search_description(webpage)).strip()
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
+        }

From df4d817bc3802e776c5056c3288953aa0ff817b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Aug 2018 02:19:30 +0700
Subject: [PATCH 016/510] [kinopoisk] Add extractor (closes #17283)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/kinopoisk.py  | 70 ++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/kinopoisk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d8e86c277..9fc1cfa70 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -520,6 +520,7 @@ from .keezmovies import KeezMoviesIE
 from .ketnet import KetnetIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
+from .kinopoisk import KinoPoiskIE
 from .keek import KeekIE
 from .konserthusetplay import KonserthusetPlayIE
 from .kontrtube import KontrTubeIE
diff --git a/youtube_dl/extractor/kinopoisk.py b/youtube_dl/extractor/kinopoisk.py
new file mode 100644
index 000000000..9e8d01f53
--- /dev/null
+++ b/youtube_dl/extractor/kinopoisk.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    dict_get,
+    int_or_none,
+)
+
+
+class KinoPoiskIE(InfoExtractor):
+    _GEO_COUNTRIES = ['RU']
+    _VALID_URL = r'https?://(?:www\.)?kinopoisk\.ru/film/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.kinopoisk.ru/film/81041/watch/',
+        'md5': '4f71c80baea10dfa54a837a46111d326',
+        'info_dict': {
+            'id': '81041',
+            'ext': 'mp4',
+            'title': 'Алеша попович и тугарин змей',
+            'description': 'md5:43787e673d68b805d0aa1df5a5aea701',
+            'thumbnail': r're:^https?://.*',
+            'duration': 4533,
+            'age_limit': 12,
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
+        'url': 'https://www.kinopoisk.ru/film/81041',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://ott-widget.kinopoisk.ru/v1/kp/', video_id,
+            query={'kpId': video_id})
+
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+\btype=["\']application/json[^>]+>(.+?)<',
+                webpage, 'data'),
+            video_id)['models']
+
+        film = data['filmStatus']
+        title = film.get('title') or film['originalTitle']
+
+        formats = self._extract_m3u8_formats(
+            data['playlistEntity']['uri'], video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        description = dict_get(
+            film, ('descriptscription', 'description',
+                   'shortDescriptscription', 'shortDescription'))
+        thumbnail = film.get('coverUrl') or film.get('posterUrl')
+        duration = int_or_none(film.get('duration'))
+        age_limit = int_or_none(film.get('restrictionAge'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From b662273989ec18e9905b67467b5feded30f19d4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Aug 2018 02:28:25 +0700
Subject: [PATCH 017/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 665503827..a4cc6e4ac 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+* [utils] Use pure browser header for User-Agent (#17236)
+
+Extractors
++ [kinopoisk] Add support for kinopoisk.ru (#17283)
++ [yourporn] Add support for yourporn.sexy (#17298)
++ [go] Add support for disneynow.go.com (#16299, #17264)
++ [6play] Add support for play.rtl.hr (#17249)
+* [anvato] Fallback to generic API key for access-key-to-API-key lookup
+  (#16788, #17254)
+* [lci] Fix extraction (#17274)
+* [bbccouk] Extend id URL regular expression (#17270)
+* [cwtv] Fix extraction (#17256)
+* [nova] Fix extraction (#17241)
++ [generic] Add support for expressen embeds
+* [raywenderlich] Adapt to site redesign (#17225)
++ [redbulltv] Add support redbull.com tv URLs (#17218)
++ [bitchute] Add support for bitchute.com (#14052)
++ [clyp] Add support for token protected media (#17184)
+* [imdb] Fix extension extraction (#17167)
+
+
 version 2018.08.04
 
 Extractors

From 4c86163b60ee3acc7f4b02c59c80d9027f378288 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Aug 2018 02:32:18 +0700
Subject: [PATCH 018/510] release 2018.08.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++++
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 20433e915..6d54efbc3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.08.04*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.08.04**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.08.22*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.08.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.08.04
+[debug] youtube-dl version 2018.08.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a4cc6e4ac..c34e9682a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.08.22
 
 Core
 * [utils] Use pure browser header for User-Agent (#17236)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4bf2ec81b..5d2a355ff 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -108,6 +108,8 @@
  - **BiliBili**
  - **BioBioChileTV**
  - **BIQLE**
+ - **BitChute**
+ - **BitChuteChannel**
  - **BleacherReport**
  - **BleacherReportCMS**
  - **blinkx**
@@ -405,6 +407,7 @@
  - **Ketnet**
  - **KhanAcademy**
  - **KickStarter**
+ - **KinoPoisk**
  - **KonserthusetPlay**
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
@@ -696,6 +699,7 @@
  - **RaiPlayLive**
  - **RaiPlayPlaylist**
  - **RayWenderlich**
+ - **RayWenderlichCourse**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
@@ -1093,6 +1097,7 @@
  - **YouNowLive**
  - **YouNowMoment**
  - **YouPorn**
+ - **YourPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
  - **youtube:channel**: YouTube.com channels
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 67394fa01..f68848389 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.08.04'
+__version__ = '2018.08.22'

From c707d2067d649f23f03cf11995d1eeb665d51838 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Aug 2018 23:18:07 +0700
Subject: [PATCH 019/510] [nova:embed] Add extractor (closes #17282)

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/nova.py       | 138 ++++++++++++++++++++---------
 2 files changed, 101 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9fc1cfa70..a8b89bcde 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -741,7 +741,10 @@ from .nonktube import NonkTubeIE
 from .noovo import NoovoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
-from .nova import NovaIE
+from .nova import (
+    NovaEmbedIE,
+    NovaIE,
+)
 from .novamov import (
     AuroraVidIE,
     CloudTimeIE,
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index edf8b82c6..80186ec50 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -6,30 +6,90 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    int_or_none,
     js_to_json,
+    qualities,
     unified_strdate,
     url_or_none,
 )
 
 
+class NovaEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://media\.cms\.nova\.cz/embed/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://media.cms.nova.cz/embed/8o0n0r?autoplay=1',
+        'md5': 'b3834f6de5401baabf31ed57456463f7',
+        'info_dict': {
+            'id': '8o0n0r',
+            'ext': 'mp4',
+            'title': '2180. díl',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 2578,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        bitrates = self._parse_json(
+            self._search_regex(
+                r'(?s)bitrates\s*=\s*({.+?})\s*;', webpage, 'formats'),
+            video_id, transform_source=js_to_json)
+
+        QUALITIES = ('lq', 'mq', 'hq', 'hd')
+        quality_key = qualities(QUALITIES)
+
+        formats = []
+        for format_id, format_list in bitrates.items():
+            if not isinstance(format_list, list):
+                continue
+            for format_url in format_list:
+                format_url = url_or_none(format_url)
+                if not format_url:
+                    continue
+                f = {
+                    'url': format_url,
+                }
+                f_id = format_id
+                for quality in QUALITIES:
+                    if '%s.mp4' % quality in format_url:
+                        f_id += '-%s' % quality
+                        f.update({
+                            'quality': quality_key(quality),
+                            'format_note': quality.upper(),
+                        })
+                        break
+                f['format_id'] = f_id
+                formats.append(f)
+        self._sort_formats(formats)
+
+        title = self._og_search_title(
+            webpage, default=None) or self._search_regex(
+            (r'<value>(?P<title>[^<]+)',
+             r'videoTitle\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
+            'title', group='value')
+        thumbnail = self._og_search_thumbnail(
+            webpage, default=None) or self._search_regex(
+            r'poster\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+            'thumbnail', fatal=False, group='value')
+        duration = int_or_none(self._search_regex(
+            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
     _VALID_URL = r'https?://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
-        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
-        'info_dict': {
-            'id': '1608920',
-            'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
-            'ext': 'flv',
-            'title': 'Duel: Michal Hrdlička a Petr Suchoň',
-            'description': 'md5:d0cc509858eee1b1374111c588c6f5d5',
-            'thumbnail': r're:^https?://.*\.(?:jpg)',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
-    }, {
         'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
         'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
         'info_dict': {
@@ -40,33 +100,6 @@ class NovaIE(InfoExtractor):
             'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
             'thumbnail': r're:^https?://.*\.(?:jpg)',
         }
-    }, {
-        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-        'info_dict': {
-            'id': '1756825',
-            'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'flv',
-            'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
-            'thumbnail': r're:^https?://.*\.(?:jpg)',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
-        'info_dict': {
-            'id': '1756858',
-            'ext': 'flv',
-            'title': 'Televizní noviny - 30. 5. 2015',
-            'thumbnail': r're:^https?://.*\.(?:jpg)',
-            'upload_date': '20150530',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
     }, {
         'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
         'info_dict': {
@@ -81,6 +114,20 @@ class NovaIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        # media.cms.nova.cz embed
+        'url': 'https://novaplus.nova.cz/porad/ulice/epizoda/18760-2180-dil',
+        'info_dict': {
+            'id': '8o0n0r',
+            'ext': 'mp4',
+            'title': '2180. díl',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 2578,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [NovaEmbedIE.ie_key()],
     }, {
         'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
         'only_matching': True,
@@ -105,6 +152,15 @@ class NovaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        # novaplus
+        embed_id = self._search_regex(
+            r'<iframe[^>]+\bsrc=["\'](?:https?:)?//media\.cms\.nova\.cz/embed/([^/?#&]+)',
+            webpage, 'embed url', default=None)
+        if embed_id:
+            return self.url_result(
+                'https://media.cms.nova.cz/embed/%s' % embed_id,
+                ie=NovaEmbedIE.ie_key(), video_id=embed_id)
+
         video_id = self._search_regex(
             [r"(?:media|video_id)\s*:\s*'(\d+)'",
              r'media=(\d+)',

From 135e6a1c107f49e7968013a57c33eea615406b96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Aug 2018 02:36:56 +0700
Subject: [PATCH 020/510] [vidzi] Add support for vidzi.nu (closes #17316)

---
 youtube_dl/extractor/vidzi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index d70283479..42ea4952c 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class VidziIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vidzi\.(?:tv|cc|si)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?vidzi\.(?:tv|cc|si|nu)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'http://vidzi.tv/cghql9yq6emu.html',
         'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',
@@ -35,6 +35,9 @@ class VidziIE(InfoExtractor):
     }, {
         'url': 'https://vidzi.si/rph9gztxj1et.html',
         'only_matching': True,
+    }, {
+        'url': 'http://vidzi.nu/cghql9yq6emu.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From beff09505cc4a7a980901c6e48adc1620aa417aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Aug 2018 04:00:35 +0700
Subject: [PATCH 021/510] [xfileshare] Add support for vidto.se (closes #17317)

---
 youtube_dl/extractor/xfileshare.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index bc3239f68..b38c7a7b3 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -23,7 +23,7 @@ class XFileShareIE(InfoExtractor):
         (r'powerwatch\.pw', 'PowerWatch'),
         (r'rapidvideo\.ws', 'Rapidvideo.ws'),
         (r'thevideobee\.to', 'TheVideoBee'),
-        (r'vidto\.me', 'Vidto'),
+        (r'vidto\.(?:me|se)', 'Vidto'),
         (r'streamin\.to', 'Streamin.To'),
         (r'xvidstage\.com', 'XVIDSTAGE'),
         (r'vidabc\.com', 'Vid ABC'),
@@ -115,7 +115,10 @@ class XFileShareIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'http://www.fastvideo.me/k8604r8nk8sn/FAST_FURIOUS_8_-_Trailer_italiano_ufficiale.mp4.html',
-        'only_matching': True
+        'only_matching': True,
+    }, {
+        'url': 'http://vidto.se/1tx1pf6t12cg.html',
+        'only_matching': True,
     }]
 
     @staticmethod

From e0b6e988710037791057cd0b65f795d44e2f534c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Aug 2018 23:12:53 +0700
Subject: [PATCH 022/510] [generic] Allow relative src for videojs embeds
 (closes #17324)

---
 youtube_dl/extractor/generic.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 83a31f3d3..096348513 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2071,6 +2071,21 @@ class GenericIE(InfoExtractor):
             },
             'playlist_count': 6,
         },
+        {
+            # videojs embed
+            'url': 'https://video.sibnet.ru/shell.php?videoid=3422904',
+            'info_dict': {
+                'id': 'shell',
+                'ext': 'mp4',
+                'title': 'Доставщик пиццы спросил разрешения сыграть на фортепиано',
+                'description': 'md5:89209cdc587dab1e4a090453dbaa2cb1',
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'expected_warnings': ['Failed to download MPD manifest'],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -3152,8 +3167,8 @@ class GenericIE(InfoExtractor):
                 sources = [sources]
             formats = []
             for source in sources:
-                src = url_or_none(source.get('src'))
-                if not src:
+                src = source.get('src')
+                if not src or not isinstance(src, compat_str):
                     continue
                 src = compat_urlparse.urljoin(url, src)
                 src_type = source.get('type')

From dac6f7654a577533ebcb825f34f0d7735274b205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Aug 2018 20:07:52 +0700
Subject: [PATCH 023/510] [tvplayhome] Add extractor (closes #17344)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/tvplay.py     | 114 ++++++++++++++++++++++++++++-
 2 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a8b89bcde..6012d0f5f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1181,6 +1181,7 @@ from .tvp import (
 from .tvplay import (
     TVPlayIE,
     ViafreeIE,
+    TVPlayHomeIE,
 )
 from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index d3adab457..8f1ff3b76 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -32,12 +32,12 @@ class TVPlayIE(InfoExtractor):
                         https?://
                             (?:www\.)?
                             (?:
-                                tvplay(?:\.skaties)?\.lv/parraides|
-                                (?:tv3play|play\.tv3)\.lt/programos|
+                                tvplay(?:\.skaties)?\.lv(?:/parraides)?|
+                                (?:tv3play|play\.tv3)\.lt(?:/programos)?|
                                 tv3play(?:\.tv3)?\.ee/sisu|
                                 (?:tv(?:3|6|8|10)play|viafree)\.se/program|
                                 (?:(?:tv3play|viasat4play|tv6play|viafree)\.no|(?:tv3play|viafree)\.dk)/programmer|
-                                play\.novatv\.bg/programi
+                                play\.nova(?:tv)?\.bg/programi
                             )
                             /(?:[^/]+/)+
                         )
@@ -203,10 +203,18 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'https://play.nova.bg/programi/zdravei-bulgariya/764300?autostart=true',
+            'only_matching': True,
+        },
         {
             'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
             'only_matching': True,
         },
+        {
+            'url': 'https://tvplay.skaties.lv/vinas-melo-labak/418113/?autostart=true',
+            'only_matching': True,
+        },
         {
             # views is null
             'url': 'http://tvplay.skaties.lv/parraides/tv3-zinas/760183',
@@ -288,6 +296,7 @@ class TVPlayIE(InfoExtractor):
                         'url': m.group('url'),
                         'app': m.group('app'),
                         'play_path': m.group('playpath'),
+                        'preference': -1,
                     })
                 else:
                     fmt.update({
@@ -447,3 +456,102 @@ class ViafreeIE(InfoExtractor):
                     'skip_rtmp': True,
                 }),
             ie=TVPlayIE.ie_key(), video_id=video_id)
+
+
+class TVPlayHomeIE(InfoExtractor):
+    _VALID_URL = r'https?://tvplay\.(?:tv3\.lt|skaties\.lv|tv3\.ee)/[^/]+/[^/?#&]+-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://tvplay.tv3.lt/aferistai-n-7/aferistai-10047125/',
+        'info_dict': {
+            'id': '366367',
+            'ext': 'mp4',
+            'title': 'Aferistai',
+            'description': 'Aferistai. Kalėdinė pasaka.',
+            'series': 'Aferistai [N-7]',
+            'season': '1 sezonas',
+            'season_number': 1,
+            'duration': 464,
+            'timestamp': 1394209658,
+            'upload_date': '20140307',
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [TVPlayIE.ie_key()],
+    }, {
+        'url': 'https://tvplay.skaties.lv/vinas-melo-labak/vinas-melo-labak-10280317/',
+        'only_matching': True,
+    }, {
+        'url': 'https://tvplay.tv3.ee/cool-d-ga-mehhikosse/cool-d-ga-mehhikosse-10044354/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'data-asset-id\s*=\s*["\'](\d{5,7})\b', webpage, 'video id',
+            default=None)
+
+        if video_id:
+            return self.url_result(
+                'mtg:%s' % video_id, ie=TVPlayIE.ie_key(), video_id=video_id)
+
+        m3u8_url = self._search_regex(
+            r'data-file\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'm3u8 url', group='url')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'data-title\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+            'title', default=None, group='value') or self._html_search_meta(
+            'title', webpage, default=None) or self._og_search_title(
+            webpage)
+
+        description = self._html_search_meta(
+            'description', webpage,
+            default=None) or self._og_search_description(webpage)
+
+        thumbnail = self._search_regex(
+            r'data-image\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'thumbnail', default=None, group='url') or self._html_search_meta(
+            'thumbnail', webpage, default=None) or self._og_search_thumbnail(
+            webpage)
+
+        duration = int_or_none(self._search_regex(
+            r'data-duration\s*=\s*["\'](\d+)', webpage, 'duration',
+            fatal=False))
+
+        season = self._search_regex(
+            (r'data-series-title\s*=\s*(["\'])[^/]+/(?P<value>(?:(?!\1).)+)\1',
+             r'\bseason\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
+            'season', default=None, group='value')
+        season_number = int_or_none(self._search_regex(
+            r'(\d+)(?:[.\s]+sezona|\s+HOOAEG)', season or '', 'season number',
+            default=None))
+        episode = self._search_regex(
+            r'(["\'])(?P<value>(?:(?!\1).)+)\1', webpage, 'episode',
+            default=None, group='value')
+        episode_number = int_or_none(self._search_regex(
+            r'(?:S[eē]rija|Osa)\s+(\d+)', episode or '', 'episode number',
+            default=None))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'season': season,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
+            'formats': formats,
+        }

From 287cf7e443814b405caac87449ee34fb055cc6b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Aug 2018 20:08:31 +0700
Subject: [PATCH 024/510] [generic] Remove unused import

---
 youtube_dl/extractor/generic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 096348513..229dfda1b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,7 +32,6 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     UnsupportedError,
-    url_or_none,
     xpath_text,
 )
 from .commonprotocols import RtmpIE

From dd88fd65a5eda12694104afa3fa891c640de71c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Aug 2018 21:41:55 +0700
Subject: [PATCH 025/510] [webofstories:playlist] Fix extraction (closes
 #16914)

---
 youtube_dl/extractor/webofstories.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 1eb1f6702..f2b8d19b4 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    orderedSet,
+)
 
 
 class WebOfStoriesIE(InfoExtractor):
@@ -133,8 +136,10 @@ class WebOfStoriesPlaylistIE(InfoExtractor):
         webpage = self._download_webpage(url, playlist_id)
 
         entries = [
-            self.url_result('http://www.webofstories.com/play/%s' % video_number, 'WebOfStories')
-            for video_number in set(re.findall(r'href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
+            self.url_result(
+                'http://www.webofstories.com/play/%s' % video_id,
+                'WebOfStories', video_id=video_id)
+            for video_id in orderedSet(re.findall(r'\bid=["\']td_(\d+)', webpage))
         ]
 
         title = self._search_regex(

From 02df41354cfae04555b3f78b2fcca3e66ec7faba Mon Sep 17 00:00:00 2001
From: Andrew Udvare <Tatsh@users.noreply.github.com>
Date: Mon, 27 Aug 2018 11:04:56 -0400
Subject: [PATCH 026/510] [bitchute] Fix extraction by pass custom User-Agent

---
 youtube_dl/extractor/bitchute.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index da263714a..446a1ab19 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -33,7 +33,9 @@ class BitChuteIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://www.bitchute.com/video/%s' % video_id, video_id)
+            'https://www.bitchute.com/video/%s' % video_id, video_id, headers={
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
+            })
 
         title = self._search_regex(
             (r'<[^>]+\bid=["\']video-title[^>]+>([^<]+)', r'<title>([^<]+)'),

From 409b9324da3ead199b26081b228b042b0b494f8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Aug 2018 02:14:47 +0700
Subject: [PATCH 027/510] [youtube:playlist] Add support for music album
 playlists (OLAK5uy_ prefix) (closes #17361)

---
 youtube_dl/extractor/youtube.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 117a57911..0442906df 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -64,7 +64,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
 
-    _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,}'
+    _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL|OLAK5uy_)[0-9A-Za-z-_]{10,}'
 
     def _set_language(self):
         self._set_cookie(
@@ -2123,7 +2123,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
                             youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
                         )
                         (
-                            (?:PL|LL|EC|UU|FL|RD|UL|TL)?[0-9A-Za-z-_]{10,}
+                            (?:PL|LL|EC|UU|FL|RD|UL|TL|OLAK5uy_)?[0-9A-Za-z-_]{10,}
                             # Top tracks, they can also include dots
                             |(?:MC)[\w\.]*
                         )
@@ -2261,6 +2261,10 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     }, {
         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
         'only_matching': True,
+    }, {
+        # music album playlist
+        'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):

From 56213aff1d8bd8f06a940fcbd65a96a3baad9d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Aug 2018 03:07:18 +0700
Subject: [PATCH 028/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c34e9682a..71174bbe6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+version <unreleased>
+
+Extractors
++ [youtube:playlist] Add support for music album playlists (OLAK5uy_ prefix)
+  (#17361)
+* [bitchute] Fix extraction by pass custom User-Agent (#17360)
+* [webofstories:playlist] Fix extraction (#16914)
++ [tvplayhome] Add support for new tvplay URLs (#17344)
++ [generic] Allow relative src for videojs embeds (#17324)
++ [xfileshare] Add support for vidto.se (#17317)
++ [vidzi] Add support for vidzi.nu (#17316)
++ [nova:embed] Add support for media.cms.nova.cz (#17282)
+
+
 version 2018.08.22
 
 Core

From eebbce5656fd7d626594aa54d2d1ec545d9ae53a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Aug 2018 03:10:09 +0700
Subject: [PATCH 029/510] release 2018.08.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6d54efbc3..0816c4f5f 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.08.22*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.08.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.08.28*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.08.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.08.22
+[debug] youtube-dl version 2018.08.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 71174bbe6..49f44a6e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.08.28
 
 Extractors
 + [youtube:playlist] Add support for music album playlists (OLAK5uy_ prefix)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5d2a355ff..5beb9bc17 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -580,6 +580,7 @@
  - **Normalboots**
  - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
+ - **NovaEmbed**
  - **nowness**
  - **nowness:playlist**
  - **nowness:series**
@@ -916,6 +917,7 @@
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
  - **TVPlayer**
+ - **TVPlayHome**
  - **Tweakers**
  - **twitch:chapter**
  - **twitch:clips**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f68848389..3e3fe1375 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.08.22'
+__version__ = '2018.08.28'

From 8959018a5f3efc02f7402c44f4d3950c0811b97a Mon Sep 17 00:00:00 2001
From: Andrew Udvare <audvare@gmail.com>
Date: Fri, 16 Mar 2018 20:11:47 -0400
Subject: [PATCH 030/510] [utils] Skip remote IP addresses non matching to
 source address' IP version (closes #13422)

---
 youtube_dl/utils.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0c830ba71..2be8c95cd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -882,7 +882,40 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
         kwargs['strict'] = True
     hc = http_class(*args, **compat_kwargs(kwargs))
     source_address = ydl_handler._params.get('source_address')
+
     if source_address is not None:
+        filter_for = socket.AF_INET if '.' in source_address else socket.AF_INET6
+        # This is to workaround _create_connection() from socket where it will try all
+        # address data from getaddrinfo() including IPv6. This filters the result from
+        # getaddrinfo() based on the source_address value.
+        # This is based on the cpython socket.create_connection() function.
+        # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
+        def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            host, port = address
+            err = None
+            addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
+            ip_addrs = [addr for addr in addrs if addr[0] == filter_for]
+            for res in ip_addrs:
+                af, socktype, proto, canonname, sa = res
+                sock = None
+                try:
+                    sock = socket.socket(af, socktype, proto)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+                        sock.settimeout(timeout)
+                    sock.bind(source_address)
+                    sock.connect(sa)
+                    err = None  # Explicitly break reference cycle
+                    return sock
+                except socket.error as _:
+                    err = _
+                    if sock is not None:
+                        sock.close()
+            if err is not None:
+                raise err
+            else:
+                raise socket.error('Unknown error occurred')
+        hc._create_connection = _create_connection
+
         sa = (source_address, 0)
         if hasattr(hc, 'source_address'):  # Python 2.7+
             hc.source_address = sa

From 9e21e6d96bed929ba57b8ce775e2a0e29e54dd60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Aug 2018 01:17:21 +0700
Subject: [PATCH 031/510] [utils] Improve remote address skipping and add
 support for python 2.6 (closes #17362)

---
 youtube_dl/utils.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2be8c95cd..bcfb72d43 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -49,7 +49,6 @@ from .compat import (
     compat_os_name,
     compat_parse_qs,
     compat_shlex_quote,
-    compat_socket_create_connection,
     compat_str,
     compat_struct_pack,
     compat_struct_unpack,
@@ -884,7 +883,6 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     source_address = ydl_handler._params.get('source_address')
 
     if source_address is not None:
-        filter_for = socket.AF_INET if '.' in source_address else socket.AF_INET6
         # This is to workaround _create_connection() from socket where it will try all
         # address data from getaddrinfo() including IPv6. This filters the result from
         # getaddrinfo() based on the source_address value.
@@ -894,7 +892,13 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
             host, port = address
             err = None
             addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
-            ip_addrs = [addr for addr in addrs if addr[0] == filter_for]
+            af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
+            ip_addrs = [addr for addr in addrs if addr[0] == af]
+            if addrs and not ip_addrs:
+                ip_version = 'v4' if af == socket.AF_INET else 'v6'
+                raise socket.error(
+                    "No remote IP%s addresses available for connect, can't use '%s' as source address"
+                    % (ip_version, source_address[0]))
             for res in ip_addrs:
                 af, socktype, proto, canonname, sa = res
                 sock = None
@@ -913,15 +917,15 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
             if err is not None:
                 raise err
             else:
-                raise socket.error('Unknown error occurred')
-        hc._create_connection = _create_connection
-
+                raise socket.error('getaddrinfo returns an empty list')
+        if hasattr(hc, '_create_connection'):
+            hc._create_connection = _create_connection
         sa = (source_address, 0)
         if hasattr(hc, 'source_address'):  # Python 2.7+
             hc.source_address = sa
         else:  # Python 2.6
             def _hc_connect(self, *args, **kwargs):
-                sock = compat_socket_create_connection(
+                sock = _create_connection(
                     (self.host, self.port), self.timeout, sa)
                 if is_https:
                     self.sock = ssl.wrap_socket(

From 73f3bdbeb4b62a1286bcd74fdd1d5741740ea845 Mon Sep 17 00:00:00 2001
From: Leonardo Taccari <iamleot@gmail.com>
Date: Thu, 30 Aug 2018 21:15:46 +0200
Subject: [PATCH 032/510] [internazionale] Fix extraction of
 non-available-abroad videos

---
 youtube_dl/extractor/internazionale.py | 27 +++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/internazionale.py b/youtube_dl/extractor/internazionale.py
index 10ba1f6cf..676e8e269 100644
--- a/youtube_dl/extractor/internazionale.py
+++ b/youtube_dl/extractor/internazionale.py
@@ -7,7 +7,7 @@ from ..utils import unified_timestamp
 
 class InternazionaleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.internazionale.it/video/2015/02/19/richard-linklater-racconta-una-scena-di-boyhood',
         'md5': '3e39d32b66882c1218e305acbf8348ca',
         'info_dict': {
@@ -23,7 +23,23 @@ class InternazionaleIE(InfoExtractor):
         'params': {
             'format': 'bestvideo',
         },
-    }
+    }, {
+        'url': 'https://www.internazionale.it/video/2018/08/29/telefono-stare-con-noi-stessi',
+        'md5': '9db8663704cab73eb972d1cee0082c79',
+        'info_dict': {
+            'id': '761344',
+            'display_id': 'telefono-stare-con-noi-stessi',
+            'ext': 'mp4',
+            'title': 'Usiamo il telefono per evitare di stare con noi stessi',
+            'description': 'md5:75ccfb0d6bcefc6e7428c68b4aa1fe44',
+            'timestamp': 1535528954,
+            'upload_date': '20180829',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -40,8 +56,13 @@ class InternazionaleIE(InfoExtractor):
             DATA_RE % 'job-id', webpage, 'video id', group='value')
         video_path = self._search_regex(
             DATA_RE % 'video-path', webpage, 'video path', group='value')
+        video_available_abroad = self._search_regex(
+            DATA_RE % 'video-available_abroad', webpage,
+            'video available aboard', default='1', group='value')
+        video_available_abroad = video_available_abroad == '1'
 
-        video_base = 'https://video.internazionale.it/%s/%s.' % (video_path, video_id)
+        video_base = 'https://video%s.internazionale.it/%s/%s.' % \
+            ('' if video_available_abroad else '-ita', video_path, video_id)
 
         formats = self._extract_m3u8_formats(
             video_base + 'm3u8', display_id, 'mp4',

From 14b7a24c19d70ec172eaedf799bcf81445ac9403 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 30 Aug 2018 14:32:35 -0500
Subject: [PATCH 033/510] [bandcamp] Extract track_number (closes #17266)

---
 youtube_dl/extractor/bandcamp.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index b8514734d..bc4c5165a 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -44,6 +44,17 @@ class BandcampIE(InfoExtractor):
             'title': 'Ben Prunty - Lanius (Battle)',
             'uploader': 'Ben Prunty',
         },
+    }, {
+        'url': 'https://relapsealumni.bandcamp.com/track/hail-to-fire',
+        'info_dict': {
+            'id': '2584466013',
+            'ext': 'mp3',
+            'title': 'Hail to Fire',
+            'track_number': 5,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -82,6 +93,7 @@ class BandcampIE(InfoExtractor):
                     'thumbnail': thumbnail,
                     'formats': formats,
                     'duration': float_or_none(data.get('duration')),
+                    'track_number': int_or_none(data.get('track_num')),
                 }
             else:
                 raise ExtractorError('No free songs found')

From 4991e16c2a62ecd1a81cbb9ff6563885c1bf6210 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Aug 2018 03:35:55 +0700
Subject: [PATCH 034/510] [bandcamp] Extract more metadata (closes #13197)

---
 youtube_dl/extractor/bandcamp.py | 218 +++++++++++++++++++------------
 1 file changed, 134 insertions(+), 84 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index bc4c5165a..f14b407dc 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import random
 import re
 import time
@@ -16,15 +15,18 @@ from ..utils import (
     int_or_none,
     KNOWN_EXTENSIONS,
     parse_filesize,
+    str_or_none,
+    try_get,
     unescapeHTML,
     update_url_query,
     unified_strdate,
+    unified_timestamp,
     url_or_none,
 )
 
 
 class BandcampIE(InfoExtractor):
-    _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>[^/?#&]+)'
+    _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<title>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
         'md5': 'c557841d5e50261777a6585648adf439',
@@ -36,24 +38,44 @@ class BandcampIE(InfoExtractor):
         },
         '_skip': 'There is a limit of 200 free downloads / month for the test song'
     }, {
+        # free download
         'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
-        'md5': '0369ace6b939f0927e62c67a1a8d9fa7',
+        'md5': '853e35bf34aa1d6fe2615ae612564b36',
         'info_dict': {
             'id': '2650410135',
             'ext': 'aiff',
             'title': 'Ben Prunty - Lanius (Battle)',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Ben Prunty',
+            'timestamp': 1396508491,
+            'upload_date': '20140403',
+            'release_date': '20140403',
+            'duration': 260.877,
+            'track': 'Lanius (Battle)',
+            'track_number': 1,
+            'track_id': '2650410135',
+            'artist': 'Ben Prunty',
+            'album': 'FTL: Advanced Edition Soundtrack',
         },
     }, {
+        # no free download, mp3 128
         'url': 'https://relapsealumni.bandcamp.com/track/hail-to-fire',
+        'md5': 'fec12ff55e804bb7f7ebeb77a800c8b7',
         'info_dict': {
             'id': '2584466013',
             'ext': 'mp3',
-            'title': 'Hail to Fire',
+            'title': 'Mastodon - Hail to Fire',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Mastodon',
+            'timestamp': 1322005399,
+            'upload_date': '20111122',
+            'release_date': '20040207',
+            'duration': 120.79,
+            'track': 'Hail to Fire',
             'track_number': 5,
-        },
-        'params': {
-            'skip_download': True,
+            'track_id': '2584466013',
+            'artist': 'Mastodon',
+            'album': 'Call of the Mastodon',
         },
     }]
 
@@ -62,19 +84,23 @@ class BandcampIE(InfoExtractor):
         title = mobj.group('title')
         webpage = self._download_webpage(url, title)
         thumbnail = self._html_search_meta('og:image', webpage, default=None)
-        m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
-        if not m_download:
-            m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage)
-            if m_trackinfo:
-                json_code = m_trackinfo.group(1)
-                data = json.loads(json_code)[0]
-                track_id = compat_str(data['id'])
 
-                if not data.get('file'):
-                    raise ExtractorError('Not streamable', video_id=track_id, expected=True)
+        track_id = None
+        track = None
+        track_number = None
+        duration = None
 
-                formats = []
-                for format_id, format_url in data['file'].items():
+        formats = []
+        track_info = self._parse_json(
+            self._search_regex(
+                r'trackinfo\s*:\s*\[\s*({.+?})\s*\]\s*,\s*?\n',
+                webpage, 'track info', default='{}'), title)
+        if track_info:
+            file_ = track_info.get('file')
+            if isinstance(file_, dict):
+                for format_id, format_url in file_.items():
+                    if not url_or_none(format_url):
+                        continue
                     ext, abr_str = format_id.split('-', 1)
                     formats.append({
                         'format_id': format_id,
@@ -84,86 +110,110 @@ class BandcampIE(InfoExtractor):
                         'acodec': ext,
                         'abr': int_or_none(abr_str),
                     })
+            track = track_info.get('title')
+            track_id = str_or_none(track_info.get('track_id') or track_info.get('id'))
+            track_number = int_or_none(track_info.get('track_num'))
+            duration = float_or_none(track_info.get('duration'))
 
-                self._sort_formats(formats)
+        def extract(key):
+            return self._search_regex(
+                r'\b%s\s*["\']?\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1' % key,
+                webpage, key, default=None, group='value')
 
-                return {
-                    'id': track_id,
-                    'title': data['title'],
-                    'thumbnail': thumbnail,
-                    'formats': formats,
-                    'duration': float_or_none(data.get('duration')),
-                    'track_number': int_or_none(data.get('track_num')),
-                }
-            else:
-                raise ExtractorError('No free songs found')
+        artist = extract('artist')
+        album = extract('album_title')
+        timestamp = unified_timestamp(
+            extract('publish_date') or extract('album_publish_date'))
+        release_date = unified_strdate(extract('album_release_date'))
 
-        download_link = m_download.group(1)
-        video_id = self._search_regex(
-            r'(?ms)var TralbumData = .*?[{,]\s*id: (?P<id>\d+),?$',
-            webpage, 'video id')
+        download_link = self._search_regex(
+            r'freeDownloadPage\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'download link', default=None, group='url')
+        if download_link:
+            track_id = self._search_regex(
+                r'(?ms)var TralbumData = .*?[{,]\s*id: (?P<id>\d+),?$',
+                webpage, 'track id')
 
-        download_webpage = self._download_webpage(
-            download_link, video_id, 'Downloading free downloads page')
+            download_webpage = self._download_webpage(
+                download_link, track_id, 'Downloading free downloads page')
 
-        blob = self._parse_json(
-            self._search_regex(
-                r'data-blob=(["\'])(?P<blob>{.+?})\1', download_webpage,
-                'blob', group='blob'),
-            video_id, transform_source=unescapeHTML)
+            blob = self._parse_json(
+                self._search_regex(
+                    r'data-blob=(["\'])(?P<blob>{.+?})\1', download_webpage,
+                    'blob', group='blob'),
+                track_id, transform_source=unescapeHTML)
 
-        info = blob['digital_items'][0]
+            info = try_get(
+                blob, (lambda x: x['digital_items'][0],
+                       lambda x: x['download_items'][0]), dict)
+            if info:
+                downloads = info.get('downloads')
+                if isinstance(downloads, dict):
+                    if not track:
+                        track = info.get('title')
+                    if not artist:
+                        artist = info.get('artist')
+                    if not thumbnail:
+                        thumbnail = info.get('thumb_url')
 
-        downloads = info['downloads']
-        track = info['title']
+                    download_formats = {}
+                    download_formats_list = blob.get('download_formats')
+                    if isinstance(download_formats_list, list):
+                        for f in blob['download_formats']:
+                            name, ext = f.get('name'), f.get('file_extension')
+                            if all(isinstance(x, compat_str) for x in (name, ext)):
+                                download_formats[name] = ext.strip('.')
 
-        artist = info.get('artist')
-        title = '%s - %s' % (artist, track) if artist else track
+                    for format_id, f in downloads.items():
+                        format_url = f.get('url')
+                        if not format_url:
+                            continue
+                        # Stat URL generation algorithm is reverse engineered from
+                        # download_*_bundle_*.js
+                        stat_url = update_url_query(
+                            format_url.replace('/download/', '/statdownload/'), {
+                                '.rand': int(time.time() * 1000 * random.random()),
+                            })
+                        format_id = f.get('encoding_name') or format_id
+                        stat = self._download_json(
+                            stat_url, track_id, 'Downloading %s JSON' % format_id,
+                            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1],
+                            fatal=False)
+                        if not stat:
+                            continue
+                        retry_url = url_or_none(stat.get('retry_url'))
+                        if not retry_url:
+                            continue
+                        formats.append({
+                            'url': self._proto_relative_url(retry_url, 'http:'),
+                            'ext': download_formats.get(format_id),
+                            'format_id': format_id,
+                            'format_note': f.get('description'),
+                            'filesize': parse_filesize(f.get('size_mb')),
+                            'vcodec': 'none',
+                        })
 
-        download_formats = {}
-        for f in blob['download_formats']:
-            name, ext = f.get('name'), f.get('file_extension')
-            if all(isinstance(x, compat_str) for x in (name, ext)):
-                download_formats[name] = ext.strip('.')
-
-        formats = []
-        for format_id, f in downloads.items():
-            format_url = f.get('url')
-            if not format_url:
-                continue
-            # Stat URL generation algorithm is reverse engineered from
-            # download_*_bundle_*.js
-            stat_url = update_url_query(
-                format_url.replace('/download/', '/statdownload/'), {
-                    '.rand': int(time.time() * 1000 * random.random()),
-                })
-            format_id = f.get('encoding_name') or format_id
-            stat = self._download_json(
-                stat_url, video_id, 'Downloading %s JSON' % format_id,
-                transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1],
-                fatal=False)
-            if not stat:
-                continue
-            retry_url = url_or_none(stat.get('retry_url'))
-            if not retry_url:
-                continue
-            formats.append({
-                'url': self._proto_relative_url(retry_url, 'http:'),
-                'ext': download_formats.get(format_id),
-                'format_id': format_id,
-                'format_note': f.get('description'),
-                'filesize': parse_filesize(f.get('size_mb')),
-                'vcodec': 'none',
-            })
         self._sort_formats(formats)
 
+        title = '%s - %s' % (artist, track) if artist else track
+
+        if not duration:
+            duration = float_or_none(self._html_search_meta(
+                'duration', webpage, default=None))
+
         return {
-            'id': video_id,
+            'id': track_id,
             'title': title,
-            'thumbnail': info.get('thumb_url') or thumbnail,
-            'uploader': info.get('artist'),
-            'artist': artist,
+            'thumbnail': thumbnail,
+            'uploader': artist,
+            'timestamp': timestamp,
+            'release_date': release_date,
+            'duration': duration,
             'track': track,
+            'track_number': track_number,
+            'track_id': track_id,
+            'artist': artist,
+            'album': album,
             'formats': formats,
         }
 

From c1a37eb24a78fe07a3ad41fc151389e444b7962b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 1 Sep 2018 00:18:17 +0200
Subject: [PATCH 035/510] [ard] Add support for Beta ARD Mediathek

Thanks to https://blog.fefe.de/?ts=a577685d for pointing out support is missing.
---
 youtube_dl/extractor/ard.py        | 62 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 63 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 23f574d36..9c6be2dd9 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -282,3 +282,65 @@ class ARDIE(InfoExtractor):
             'upload_date': upload_date,
             'thumbnail': thumbnail,
         }
+
+
+class ARDBetaMediathekIE(InfoExtractor):
+    _VALID_URL = r'https://beta\.ardmediathek\.de/[a-z]+/player/(?P<video_id>[a-zA-Z0-9]+)/(?P<display_id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://beta.ardmediathek.de/ard/player/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE/die-robuste-roswita',
+        'md5': '2d02d996156ea3c397cfc5036b5d7f8f',
+        'info_dict': {
+            'display_id': 'die-robuste-roswita',
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
+            'title': 'Tatort: Die robuste Roswita',
+            'description': r're:^Der Mord.*trüber ist als die Ilm.',
+            'duration': 5316,
+            'thumbnail': 'https://img.ardmediathek.de/standard/00/55/43/59/34/-1774185891/16x9/960?mandant=ard',
+            'upload_date': '20180826',
+            'ext': 'mp4',
+        },
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+        data_json = self._search_regex(r'window\.__APOLLO_STATE__\s*=\s*(\{.*);', webpage, 'json')
+        data = self._parse_json(data_json, display_id)
+
+        res = {
+            'id': video_id,
+            'display_id': display_id,
+        }
+        formats = []
+        for widget in data.values():
+            if widget.get('_geoblocked'):
+                raise ExtractorError('This video is not available due to geoblocking', expected=True)
+
+            if '_duration' in widget:
+                res['duration'] = widget['_duration']
+            if 'clipTitle' in widget:
+                res['title'] = widget['clipTitle']
+            if '_previewImage' in widget:
+                res['thumbnail'] = widget['_previewImage']
+            if 'broadcastedOn' in widget:
+                res['upload_date'] = unified_strdate(widget['broadcastedOn'])
+            if 'synopsis' in widget:
+                res['description'] = widget['synopsis']
+            if '_subtitleUrl' in widget:
+                res['subtitles'] = {'de': [{
+                    'ext': 'ttml',
+                    'url': widget['_subtitleUrl'],
+                }]}
+            if '_quality' in widget:
+                formats.append({
+                    'format_id': widget['_quality'],
+                    'url': widget['_stream']['json'][0],
+                })
+
+        self._sort_formats(formats)
+        res['formats'] = formats
+
+        return res
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6012d0f5f..995af9988 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -54,6 +54,7 @@ from .appletrailers import (
 from .archiveorg import ArchiveOrgIE
 from .arkena import ArkenaIE
 from .ard import (
+    ARDBetaMediathekIE,
     ARDIE,
     ARDMediathekIE,
 )

From 2b83da246375e6b37884aad9fe22ea8e461e20b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 1 Sep 2018 00:45:36 +0200
Subject: [PATCH 036/510] [ard] Better format handling

Skip f4m, doesn't work (yet); correctly extract m3u8, and prefer plain HTTP files.
---
 youtube_dl/extractor/ard.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 9c6be2dd9..cff8ca4a5 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -335,10 +335,24 @@ class ARDBetaMediathekIE(InfoExtractor):
                     'url': widget['_subtitleUrl'],
                 }]}
             if '_quality' in widget:
-                formats.append({
-                    'format_id': widget['_quality'],
-                    'url': widget['_stream']['json'][0],
-                })
+                format_url = widget['_stream']['json'][0]
+
+                if format_url.endswith('.f4m'):
+                    # Skip f4m - these URLs just return a 403
+                    formats.append({
+                        'format_id': 'f4m-' + widget['_quality'],
+                        'url': format_url,
+                        'preference': -1001,
+                    })
+                elif format_url.endswith('m3u8'):
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                else:
+                    formats.append({
+                        'format_id': 'http-' + widget['_quality'],
+                        'url': format_url,
+                        'preference': 10,  # Plain HTTP, that's nice
+                    })
 
         self._sort_formats(formats)
         res['formats'] = formats

From ed6919e7371b3da66c10e4c5768816e81f4c5db3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 1 Sep 2018 01:59:13 +0200
Subject: [PATCH 037/510] [ard] beta mediathek: make regexp for JSON more
 robust

---
 youtube_dl/extractor/ard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index cff8ca4a5..dcb347849 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -307,7 +307,7 @@ class ARDBetaMediathekIE(InfoExtractor):
         display_id = mobj.group('display_id')
 
         webpage = self._download_webpage(url, display_id)
-        data_json = self._search_regex(r'window\.__APOLLO_STATE__\s*=\s*(\{.*);', webpage, 'json')
+        data_json = self._search_regex(r'window\.__APOLLO_STATE__\s*=\s*(\{.*);\n', webpage, 'json')
         data = self._parse_json(data_json, display_id)
 
         res = {

From 54a5be4dba3560ff51f98865b5598d361a878e82 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Sep 2018 08:16:28 +0100
Subject: [PATCH 038/510] [crunchyroll] parse vilos media data(closes #17343)

---
 youtube_dl/extractor/crunchyroll.py | 205 ++++++++++++++++------------
 youtube_dl/extractor/vrv.py         |  48 ++++---
 2 files changed, 141 insertions(+), 112 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 463f995c7..4ed458372 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -7,7 +7,7 @@ import zlib
 
 from hashlib import sha1
 from math import pow, sqrt, floor
-from .common import InfoExtractor
+from .vrv import VRVIE
 from ..compat import (
     compat_b64decode,
     compat_etree_fromstring,
@@ -18,6 +18,8 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
+    extract_attributes,
+    float_or_none,
     intlist_to_bytes,
     int_or_none,
     lowercase_escape,
@@ -26,14 +28,13 @@ from ..utils import (
     unified_strdate,
     urlencode_postdata,
     xpath_text,
-    extract_attributes,
 )
 from ..aes import (
     aes_cbc_decrypt,
 )
 
 
-class CrunchyrollBaseIE(InfoExtractor):
+class CrunchyrollBaseIE(VRVIE):
     _LOGIN_URL = 'https://www.crunchyroll.com/login'
     _LOGIN_FORM = 'login_form'
     _NETRC_MACHINE = 'crunchyroll'
@@ -148,7 +149,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'ext': 'mp4',
             'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
             'description': 'md5:2d17137920c64f2f49981a7797d275ef',
-            'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Yomiuri Telecasting Corporation (YTV)',
             'upload_date': '20131013',
             'url': 're:(?!.*&amp)',
@@ -221,7 +222,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         'info_dict': {
             'id': '535080',
             'ext': 'mp4',
-            'title': '11eyes Episode 1 – Piros éjszaka - Red Night',
+            'title': '11eyes Episode 1 – Red Night ~ Piros éjszaka',
             'description': 'Kakeru and Yuka are thrown into an alternate nightmarish world they call "Red Night".',
             'uploader': 'Marvelous AQL Inc.',
             'upload_date': '20091021',
@@ -437,13 +438,18 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
             self.raise_login_required()
 
+        media = self._parse_json(self._search_regex(
+            r'vilos\.config\.media\s*=\s*({.+?});',
+            webpage, 'vilos media', default='{}'), video_id)
+        media_metadata = media.get('metadata') or {}
+
         video_title = self._html_search_regex(
             r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
             webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
-        video_description = self._parse_json(self._html_search_regex(
+        video_description = (self._parse_json(self._html_search_regex(
             r'<script[^>]*>\s*.+?\[media_id=%s\].+?({.+?"description"\s*:.+?})\);' % video_id,
-            webpage, 'description', default='{}'), video_id).get('description')
+            webpage, 'description', default='{}'), video_id) or media_metadata).get('description')
         if video_description:
             video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
         video_upload_date = self._html_search_regex(
@@ -456,91 +462,99 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             [r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', r'<div>\s*Publisher:\s*<span>\s*(.+?)\s*</span>\s*</div>'],
             webpage, 'video_uploader', fatal=False)
 
-        available_fmts = []
-        for a, fmt in re.findall(r'(<a[^>]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage):
-            attrs = extract_attributes(a)
-            href = attrs.get('href')
-            if href and '/freetrial' in href:
-                continue
-            available_fmts.append(fmt)
-        if not available_fmts:
-            for p in (r'token=["\']showmedia\.([0-9]{3,4})p"', r'showmedia\.([0-9]{3,4})p'):
-                available_fmts = re.findall(p, webpage)
-                if available_fmts:
-                    break
-        video_encode_ids = []
         formats = []
-        for fmt in available_fmts:
-            stream_quality, stream_format = self._FORMAT_IDS[fmt]
-            video_format = fmt + 'p'
-            stream_infos = []
-            streamdata = self._call_rpc_api(
-                'VideoPlayer_GetStandardConfig', video_id,
-                'Downloading media info for %s' % video_format, data={
-                    'media_id': video_id,
-                    'video_format': stream_format,
-                    'video_quality': stream_quality,
-                    'current_page': url,
-                })
-            if streamdata is not None:
-                stream_info = streamdata.find('./{default}preload/stream_info')
+        for stream in media.get('streams', []):
+            formats.extend(self._extract_vrv_formats(
+                stream.get('url'), video_id, stream.get('format'),
+                stream.get('audio_lang'), stream.get('hardsub_lang')))
+        if not formats:
+            available_fmts = []
+            for a, fmt in re.findall(r'(<a[^>]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage):
+                attrs = extract_attributes(a)
+                href = attrs.get('href')
+                if href and '/freetrial' in href:
+                    continue
+                available_fmts.append(fmt)
+            if not available_fmts:
+                for p in (r'token=["\']showmedia\.([0-9]{3,4})p"', r'showmedia\.([0-9]{3,4})p'):
+                    available_fmts = re.findall(p, webpage)
+                    if available_fmts:
+                        break
+            if not available_fmts:
+                available_fmts = self._FORMAT_IDS.keys()
+            video_encode_ids = []
+
+            for fmt in available_fmts:
+                stream_quality, stream_format = self._FORMAT_IDS[fmt]
+                video_format = fmt + 'p'
+                stream_infos = []
+                streamdata = self._call_rpc_api(
+                    'VideoPlayer_GetStandardConfig', video_id,
+                    'Downloading media info for %s' % video_format, data={
+                        'media_id': video_id,
+                        'video_format': stream_format,
+                        'video_quality': stream_quality,
+                        'current_page': url,
+                    })
+                if streamdata is not None:
+                    stream_info = streamdata.find('./{default}preload/stream_info')
+                    if stream_info is not None:
+                        stream_infos.append(stream_info)
+                stream_info = self._call_rpc_api(
+                    'VideoEncode_GetStreamInfo', video_id,
+                    'Downloading stream info for %s' % video_format, data={
+                        'media_id': video_id,
+                        'video_format': stream_format,
+                        'video_encode_quality': stream_quality,
+                    })
                 if stream_info is not None:
                     stream_infos.append(stream_info)
-            stream_info = self._call_rpc_api(
-                'VideoEncode_GetStreamInfo', video_id,
-                'Downloading stream info for %s' % video_format, data={
-                    'media_id': video_id,
-                    'video_format': stream_format,
-                    'video_encode_quality': stream_quality,
-                })
-            if stream_info is not None:
-                stream_infos.append(stream_info)
-            for stream_info in stream_infos:
-                video_encode_id = xpath_text(stream_info, './video_encode_id')
-                if video_encode_id in video_encode_ids:
-                    continue
-                video_encode_ids.append(video_encode_id)
+                for stream_info in stream_infos:
+                    video_encode_id = xpath_text(stream_info, './video_encode_id')
+                    if video_encode_id in video_encode_ids:
+                        continue
+                    video_encode_ids.append(video_encode_id)
 
-                video_file = xpath_text(stream_info, './file')
-                if not video_file:
-                    continue
-                if video_file.startswith('http'):
-                    formats.extend(self._extract_m3u8_formats(
-                        video_file, video_id, 'mp4', entry_protocol='m3u8_native',
-                        m3u8_id='hls', fatal=False))
-                    continue
-
-                video_url = xpath_text(stream_info, './host')
-                if not video_url:
-                    continue
-                metadata = stream_info.find('./metadata')
-                format_info = {
-                    'format': video_format,
-                    'height': int_or_none(xpath_text(metadata, './height')),
-                    'width': int_or_none(xpath_text(metadata, './width')),
-                }
-
-                if '.fplive.net/' in video_url:
-                    video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
-                    parsed_video_url = compat_urlparse.urlparse(video_url)
-                    direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
-                        netloc='v.lvlt.crcdn.net',
-                        path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
-                    if self._is_valid_url(direct_video_url, video_id, video_format):
-                        format_info.update({
-                            'format_id': 'http-' + video_format,
-                            'url': direct_video_url,
-                        })
-                        formats.append(format_info)
+                    video_file = xpath_text(stream_info, './file')
+                    if not video_file:
+                        continue
+                    if video_file.startswith('http'):
+                        formats.extend(self._extract_m3u8_formats(
+                            video_file, video_id, 'mp4', entry_protocol='m3u8_native',
+                            m3u8_id='hls', fatal=False))
                         continue
 
-                format_info.update({
-                    'format_id': 'rtmp-' + video_format,
-                    'url': video_url,
-                    'play_path': video_file,
-                    'ext': 'flv',
-                })
-                formats.append(format_info)
+                    video_url = xpath_text(stream_info, './host')
+                    if not video_url:
+                        continue
+                    metadata = stream_info.find('./metadata')
+                    format_info = {
+                        'format': video_format,
+                        'height': int_or_none(xpath_text(metadata, './height')),
+                        'width': int_or_none(xpath_text(metadata, './width')),
+                    }
+
+                    if '.fplive.net/' in video_url:
+                        video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
+                        parsed_video_url = compat_urlparse.urlparse(video_url)
+                        direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
+                            netloc='v.lvlt.crcdn.net',
+                            path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
+                        if self._is_valid_url(direct_video_url, video_id, video_format):
+                            format_info.update({
+                                'format_id': 'http-' + video_format,
+                                'url': direct_video_url,
+                            })
+                            formats.append(format_info)
+                            continue
+
+                    format_info.update({
+                        'format_id': 'rtmp-' + video_format,
+                        'url': video_url,
+                        'play_path': video_file,
+                        'ext': 'flv',
+                    })
+                    formats.append(format_info)
         self._sort_formats(formats, ('height', 'width', 'tbr', 'fps'))
 
         metadata = self._call_rpc_api(
@@ -549,7 +563,17 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'media_id': video_id,
             })
 
-        subtitles = self.extract_subtitles(video_id, webpage)
+        subtitles = {}
+        for subtitle in media.get('subtitles', []):
+            subtitle_url = subtitle.get('url')
+            if not subtitle_url:
+                continue
+            subtitles.setdefault(subtitle.get('language', 'enUS'), []).append({
+                'url': subtitle_url,
+                'ext': subtitle.get('format', 'ass'),
+            })
+        if not subtitles:
+            subtitles = self.extract_subtitles(video_id, webpage)
 
         # webpage provide more accurate data than series_title from XML
         series = self._html_search_regex(
@@ -557,8 +581,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             webpage, 'series', fatal=False)
         season = xpath_text(metadata, 'series_title')
 
-        episode = xpath_text(metadata, 'episode_title')
-        episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
+        episode = xpath_text(metadata, 'episode_title') or media_metadata.get('title')
+        episode_number = int_or_none(xpath_text(metadata, 'episode_number') or media_metadata.get('episode_number'))
 
         season_number = int_or_none(self._search_regex(
             r'(?s)<h\d[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h\d>\s*<h4>\s*Season (\d+)',
@@ -568,7 +592,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'id': video_id,
             'title': video_title,
             'description': video_description,
-            'thumbnail': xpath_text(metadata, 'episode_image_url'),
+            'duration': float_or_none(media_metadata.get('duration'), 1000),
+            'thumbnail': xpath_text(metadata, 'episode_image_url') or media_metadata.get('thumbnail', {}).get('url'),
             'uploader': video_uploader,
             'upload_date': video_upload_date,
             'series': series,
diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 64b13f0ed..921e9e172 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -72,7 +72,7 @@ class VRVBaseIE(InfoExtractor):
 class VRVIE(VRVBaseIE):
     IE_NAME = 'vrv'
     _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
         'info_dict': {
             'id': 'GR9PNZ396',
@@ -85,7 +85,28 @@ class VRVIE(VRVBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }]
+
+    def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang):
+        if not url or stream_format not in ('hls', 'dash'):
+            return []
+        stream_id = hardsub_lang or audio_lang
+        format_id = '%s-%s' % (stream_format, stream_id)
+        if stream_format == 'hls':
+            adaptive_formats = self._extract_m3u8_formats(
+                url, video_id, 'mp4', m3u8_id=format_id,
+                note='Downloading %s m3u8 information' % stream_id,
+                fatal=False)
+        elif stream_format == 'dash':
+            adaptive_formats = self._extract_mpd_formats(
+                url, video_id, mpd_id=format_id,
+                note='Downloading %s MPD information' % stream_id,
+                fatal=False)
+        if audio_lang:
+            for f in adaptive_formats:
+                if f.get('acodec') != 'none':
+                    f['language'] = audio_lang
+        return adaptive_formats
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -115,26 +136,9 @@ class VRVIE(VRVBaseIE):
         for stream_type, streams in streams_json.get('streams', {}).items():
             if stream_type in ('adaptive_hls', 'adaptive_dash'):
                 for stream in streams.values():
-                    stream_url = stream.get('url')
-                    if not stream_url:
-                        continue
-                    stream_id = stream.get('hardsub_locale') or audio_locale
-                    format_id = '%s-%s' % (stream_type.split('_')[1], stream_id)
-                    if stream_type == 'adaptive_hls':
-                        adaptive_formats = self._extract_m3u8_formats(
-                            stream_url, video_id, 'mp4', m3u8_id=format_id,
-                            note='Downloading %s m3u8 information' % stream_id,
-                            fatal=False)
-                    else:
-                        adaptive_formats = self._extract_mpd_formats(
-                            stream_url, video_id, mpd_id=format_id,
-                            note='Downloading %s MPD information' % stream_id,
-                            fatal=False)
-                    if audio_locale:
-                        for f in adaptive_formats:
-                            if f.get('acodec') != 'none':
-                                f['language'] = audio_locale
-                    formats.extend(adaptive_formats)
+                    formats.extend(self._extract_vrv_formats(
+                        stream.get('url'), video_id, stream_type.split('_')[1],
+                        audio_locale, stream.get('hardsub_locale')))
         self._sort_formats(formats)
 
         subtitles = {}

From 7f2611cb5b086fbdb6281779a96fcb75a594e68c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Sep 2018 08:40:38 +0100
Subject: [PATCH 039/510] [ard] extract f4m formats

---
 youtube_dl/extractor/ard.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index dcb347849..194a369c0 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -338,12 +338,9 @@ class ARDBetaMediathekIE(InfoExtractor):
                 format_url = widget['_stream']['json'][0]
 
                 if format_url.endswith('.f4m'):
-                    # Skip f4m - these URLs just return a 403
-                    formats.append({
-                        'format_id': 'f4m-' + widget['_quality'],
-                        'url': format_url,
-                        'preference': -1001,
-                    })
+                    formats.extend(self._extract_f4m_formats(
+                        format_url + '?hdcore=3.11.0',
+                        video_id, f4m_id='hds', fatal=False))
                 elif format_url.endswith('m3u8'):
                     formats.extend(self._extract_m3u8_formats(
                         format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))

From 462799588288b9ac8280dc361a9efcced8cb29e1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Sep 2018 10:04:10 +0100
Subject: [PATCH 040/510] [crunchyroll] limit VRVIE inheritance to
 CrunchyrollIE

---
 youtube_dl/extractor/crunchyroll.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4ed458372..ba8b9fa7e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -7,6 +7,7 @@ import zlib
 
 from hashlib import sha1
 from math import pow, sqrt, floor
+from .common import InfoExtractor
 from .vrv import VRVIE
 from ..compat import (
     compat_b64decode,
@@ -34,7 +35,7 @@ from ..aes import (
 )
 
 
-class CrunchyrollBaseIE(VRVIE):
+class CrunchyrollBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.crunchyroll.com/login'
     _LOGIN_FORM = 'login_form'
     _NETRC_MACHINE = 'crunchyroll'
@@ -140,7 +141,8 @@ class CrunchyrollBaseIE(VRVIE):
             parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
 
-class CrunchyrollIE(CrunchyrollBaseIE):
+class CrunchyrollIE(CrunchyrollBaseIE, VRVIE):
+    IE_NAME = 'crunchyroll'
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',

From 4d59db5b900fa8f1416cb33328f2f12daf9946c5 Mon Sep 17 00:00:00 2001
From: Gorfiend <Gorfiend@users.noreply.github.com>
Date: Sat, 1 Sep 2018 05:04:45 -0400
Subject: [PATCH 041/510] [niconico] Fix extraction on python3 (closes #17393)

---
 youtube_dl/extractor/niconico.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index dbe871f16..76b412ff1 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -252,7 +252,7 @@ class NiconicoIE(InfoExtractor):
                     },
                     'timing_constraint': 'unlimited'
                 }
-            }))
+            }).encode())
 
         resolution = video_quality.get('resolution', {})
 

From 0b87e8845328fd93a6f0fd873f5394f63c9ce64f Mon Sep 17 00:00:00 2001
From: LangerJan <LangerJan@users.noreply.github.com>
Date: Sat, 1 Sep 2018 11:42:30 +0200
Subject: [PATCH 042/510] [ard] Add support for one.ard.de

---
 youtube_dl/extractor/ard.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 194a369c0..6bf8f61eb 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -21,7 +21,7 @@ from ..compat import compat_etree_fromstring
 
 class ARDMediathekIE(InfoExtractor):
     IE_NAME = 'ARD:mediathek'
-    _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
+    _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
 
     _TESTS = [{
         # available till 26.07.2022
@@ -37,6 +37,9 @@ class ARDMediathekIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
+        'only_matching': True,
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',

From 7bbc1b189aa7b0cea5336964a2acdca4001c1566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Sep 2018 18:36:18 +0700
Subject: [PATCH 043/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 49f44a6e6..7585f06e9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
+* [utils] Skip remote IP addresses non matching to source address' IP version
+  when creating a connection (#13422, #17362)
+
+Extractors
++ [ard] Add support for one.ard.de (#17397)
+* [niconico] Fix extraction on python3 (#17393, #17407)
+* [ard] Extract f4m formats
+* [crunchyroll] Parse vilos media data (#17343)
++ [ard] Add support for Beta ARD Mediathek
++ [bandcamp] Extract more metadata (#13197)
+* [internazionale] Fix extraction of non-available-abroad videos (#17386)
+
+
 version 2018.08.28
 
 Extractors

From 27d8e089a21561480f5261f77665604bba2f5901 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Sep 2018 18:40:23 +0700
Subject: [PATCH 044/510] release 2018.09.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 0816c4f5f..16d8f36d7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.08.28*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.08.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.08.28
+[debug] youtube-dl version 2018.09.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7585f06e9..4632133a3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.09.01
 
 Core
 * [utils] Skip remote IP addresses non matching to source address' IP version
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5beb9bc17..f0d6be901 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -56,6 +56,7 @@
  - **archive.org**: archive.org videos
  - **ARD**
  - **ARD:mediathek**
+ - **ARDBetaMediathek**
  - **Arkena**
  - **arte.tv**
  - **arte.tv:+7**
@@ -191,7 +192,7 @@
  - **Crackle**
  - **Criterion**
  - **CrooksAndLiars**
- - **Crunchyroll**
+ - **crunchyroll**
  - **crunchyroll:playlist**
  - **CSNNE**
  - **CSpan**: C-SPAN
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e3fe1375..e655e0050 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.08.28'
+__version__ = '2018.09.01'

From 3d08f63dc57f000384703f26b7dcb4b683e18c05 Mon Sep 17 00:00:00 2001
From: Mohammed Yaseen Mowzer <yaseen@mowzer.co.za>
Date: Thu, 14 Jun 2018 17:12:33 +0200
Subject: [PATCH 045/510] [generic] Skip unsuccessful jwplayer extraction
 (closes #16735)

---
 youtube_dl/extractor/generic.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 229dfda1b..1db154c4f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3150,9 +3150,13 @@ class GenericIE(InfoExtractor):
         jwplayer_data = self._find_jwplayer_data(
             webpage, video_id, transform_source=js_to_json)
         if jwplayer_data:
-            info = self._parse_jwplayer_data(
-                jwplayer_data, video_id, require_title=False, base_url=url)
-            return merge_dicts(info, info_dict)
+            try:
+                info = self._parse_jwplayer_data(
+                    jwplayer_data, video_id, require_title=False, base_url=url)
+                return merge_dicts(info, info_dict)
+            except ExtractorError:
+                # See https://github.com/rg3/youtube-dl/pull/16735
+                pass
 
         # Video.js embed
         mobj = re.search(

From 0a9a8118ce834c206434df04c18187934acbf608 Mon Sep 17 00:00:00 2001
From: Hormoz K <hkheradm@gmail.com>
Date: Sat, 4 Aug 2018 09:47:58 -0400
Subject: [PATCH 046/510] [radiojavan] Fix extraction

---
 youtube_dl/extractor/radiojavan.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index a53ad97a5..4124bcd45 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -6,11 +6,13 @@ from .common import InfoExtractor
 from ..utils import (
     unified_strdate,
     str_to_int,
+    urlencode_postdata,
 )
 
 
 class RadioJavanIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
+    _HOST_TRACKER_URL = 'https://www.radiojavan.com/videos/video_host'
     _TEST = {
         'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam',
         'md5': 'e85208ffa3ca8b83534fca9fe19af95b',
@@ -31,8 +33,18 @@ class RadioJavanIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        download_host = self._download_json(
+            self._HOST_TRACKER_URL,
+            video_id,
+            data=urlencode_postdata({'id': video_id}),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Referer': url,
+            }
+        )['host']
+
         formats = [{
-            'url': 'https://media.rdjavan.com/media/music_video/%s' % video_path,
+            'url': '%s/%s' % (download_host, video_path),
             'format_id': '%sp' % height,
             'height': int(height),
         } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]

From 93284ff2ea4eb84c25b8d496012398a056ba89ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Sep 2018 02:53:26 +0700
Subject: [PATCH 047/510] [radiojavan] Improve extraction (closes #17151)

---
 youtube_dl/extractor/radiojavan.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index 4124bcd45..3f74f0c01 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -4,15 +4,16 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
+    parse_resolution,
     str_to_int,
+    unified_strdate,
     urlencode_postdata,
+    urljoin,
 )
 
 
 class RadioJavanIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
-    _HOST_TRACKER_URL = 'https://www.radiojavan.com/videos/video_host'
     _TEST = {
         'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam',
         'md5': 'e85208ffa3ca8b83534fca9fe19af95b',
@@ -31,23 +32,26 @@ class RadioJavanIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
         download_host = self._download_json(
-            self._HOST_TRACKER_URL,
-            video_id,
+            'https://www.radiojavan.com/videos/video_host', video_id,
             data=urlencode_postdata({'id': video_id}),
             headers={
                 'Content-Type': 'application/x-www-form-urlencoded',
                 'Referer': url,
-            }
-        )['host']
+            }).get('host', 'https://host1.rjmusicmedia.com')
 
-        formats = [{
-            'url': '%s/%s' % (download_host, video_path),
-            'format_id': '%sp' % height,
-            'height': int(height),
-        } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for format_id, _, video_path in re.findall(
+                r'RJ\.video(?P<format_id>\d+[pPkK])\s*=\s*(["\'])(?P<url>(?:(?!\2).)+)\2',
+                webpage):
+            f = parse_resolution(format_id)
+            f.update({
+                'url': urljoin(download_host, video_path),
+                'format_id': format_id,
+            })
+            formats.append(f)
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From aa1d5eb9053b4c39661415ef411fb3746ee11f41 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 4 Sep 2018 10:37:11 +0100
Subject: [PATCH 048/510] [slideslive] make the check for video_service_name
 case-insensitive(closes #17429)

---
 youtube_dl/extractor/slideslive.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/slideslive.py b/youtube_dl/extractor/slideslive.py
index 104576033..ed84322c5 100644
--- a/youtube_dl/extractor/slideslive.py
+++ b/youtube_dl/extractor/slideslive.py
@@ -8,6 +8,7 @@ from ..utils import ExtractorError
 class SlidesLiveIE(InfoExtractor):
     _VALID_URL = r'https?://slideslive\.com/(?P<id>[0-9]+)'
     _TESTS = [{
+        # video_service_name = YOUTUBE
         'url': 'https://slideslive.com/38902413/gcc-ia16-backend',
         'md5': 'b29fcd6c6952d0c79c5079b0e7a07e6f',
         'info_dict': {
@@ -19,14 +20,18 @@ class SlidesLiveIE(InfoExtractor):
             'uploader_id': 'UC62SdArr41t_-_fX40QCLRw',
             'upload_date': '20170925',
         }
+    }, {
+        # video_service_name = youtube
+        'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(
             url, video_id, headers={'Accept': 'application/json'})
-        service_name = video_data['video_service_name']
-        if service_name == 'YOUTUBE':
+        service_name = video_data['video_service_name'].lower()
+        if service_name == 'youtube':
             yt_video_id = video_data['video_service_id']
             return self.url_result(yt_video_id, 'Youtube', video_id=yt_video_id)
         else:

From 09322cccdb655954c63724459556bd3ebb585d48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Sep 2018 00:22:30 +0700
Subject: [PATCH 049/510] [iprima] Confirm adult check (closes #17437)

---
 youtube_dl/extractor/iprima.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index a29e6a5ba..3c4b7e48b 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -38,6 +38,8 @@ class IPrimaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        self._set_cookie('play.iprima.cz', 'ott_adult_confirmed', '1')
+
         webpage = self._download_webpage(url, video_id)
 
         video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')

From 2d4fe594c64249c263d739c8094b445c67fa09f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Sep 2018 00:51:20 +0700
Subject: [PATCH 050/510] [pornhub:uservideos] Add support for new URLs (closes
 #17388)

---
 youtube_dl/extractor/pornhub.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index ffc4405a8..6782848d9 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -254,7 +254,7 @@ class PornHubIE(InfoExtractor):
         self._sort_formats(formats)
 
         video_uploader = self._html_search_regex(
-            r'(?s)From:&nbsp;.+?<(?:a\b[^>]+\bhref=["\']/(?:user|channel)s/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
+            r'(?s)From:&nbsp;.+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
 
         view_count = self._extract_count(
@@ -346,7 +346,7 @@ class PornHubPlaylistIE(PornHubPlaylistBaseIE):
 
 
 class PornHubUserVideosIE(PornHubPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.com/(?:user|channel)s/(?P<id>[^/]+)/videos'
+    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.com/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos'
     _TESTS = [{
         'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
         'info_dict': {
@@ -378,6 +378,12 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
     }, {
         'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
         'only_matching': True,
+    }, {
+        'url': 'https://www.pornhub.com/model/jayndrea/videos/upload',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9a47fa35dd9dd2d53f4d3f088811ea29295991e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 03:36:10 +0700
Subject: [PATCH 051/510] [youtube] Fix extraction (closes #17457, closes
 #17464)

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0442906df..27047425d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1178,7 +1178,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
             (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\('),
+             r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*c\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
         jsi = JSInterpreter(jscode)

From a41a506077936b2574cbb34afa83f8215accc3a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 03:40:06 +0700
Subject: [PATCH 052/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 4632133a3..b378f0a0e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+version <unreleased>
+
+Extractors
+* [youtube] Fix extraction (#17457, #17464)
++ [pornhub:uservideos] Add support for new URLs (#17388)
+* [iprima] Confirm adult check (#17437)
+* [slideslive] Make check for video service name case-insensitive (#17429)
+* [radiojavan] Fix extraction (#17151)
+* [generic] Skip unsuccessful jwplayer extraction (#16735)
+
+
 version 2018.09.01
 
 Core

From ad98d2eb748b593dc044296493904607c3058e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 03:42:28 +0700
Subject: [PATCH 053/510] release 2018.09.08

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 16d8f36d7..2d67247e6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.08*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.08**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.09.01
+[debug] youtube-dl version 2018.09.08
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index b378f0a0e..ac95c9b71 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.09.08
 
 Extractors
 * [youtube] Fix extraction (#17457, #17464)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e655e0050..716d9ffe0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.09.01'
+__version__ = '2018.09.08'

From d0c5fabc1215fddd354fd91cd65082e45df1165e Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Sat, 8 Sep 2018 09:44:06 +0200
Subject: [PATCH 054/510] [nbc] Fix extraction of percent encoded URLs (closes
 #17374)

---
 youtube_dl/extractor/nbc.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index c843f8649..765c46fd2 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -7,6 +7,7 @@ import re
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
 from .adobepass import AdobePassIE
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     find_xpath_attr,
     smuggle_url,
@@ -75,11 +76,16 @@ class NBCIE(AdobePassIE):
             'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
             'only_matching': True,
         },
+        {
+            # Percent escaped url
+            'url': 'https://www.nbc.com/up-all-night/video/day-after-valentine%27s-day/n2189',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
         permalink, video_id = re.match(self._VALID_URL, url).groups()
-        permalink = 'http' + permalink
+        permalink = 'http' + compat_urllib_parse_unquote(permalink)
         response = self._download_json(
             'https://api.nbc.com/v3/videos', video_id, query={
                 'filter[permalink]': permalink,

From d0de6a287ac64b52630e5b21c3db52a4312675fd Mon Sep 17 00:00:00 2001
From: Jens Rutschmann <N/A>
Date: Sat, 1 Sep 2018 19:43:34 +0200
Subject: [PATCH 055/510] [tele5] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tele5.py      | 39 ++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/tele5.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 995af9988..7dc569724 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1086,6 +1086,7 @@ from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
+from .tele5 import Tele5IE
 from .tele13 import Tele13IE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
diff --git a/youtube_dl/extractor/tele5.py b/youtube_dl/extractor/tele5.py
new file mode 100644
index 000000000..cfa8d2475
--- /dev/null
+++ b/youtube_dl/extractor/tele5.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .nexx import NexxIE
+
+
+class Tele5IE(InfoExtractor):
+    _VALID_URL = r'https://www\.tele5\.de/(?:mediathek/filme-online/videos\?vid=|tv/)(?P<display_id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://www.tele5.de/mediathek/filme-online/videos?vid=1550589',
+        'info_dict': {
+            'id': '1550589',
+            'ext': 'mp4',
+            'upload_date': '20180822',
+            'timestamp': 1534927316,
+            'title': 'SchleFaZ: Atomic Shark',
+        }
+    }, {
+        'url': 'https://www.tele5.de/tv/dark-matter/videos',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_regex(
+            r'id\s*=\s*["\']video-player["\']\s*data-id\s*=\s*["\']([0-9]+)["\']',
+            webpage, 'video_id')
+
+        return self.url_result(
+            'https://api.nexx.cloud/v3/759/videos/byid/%s' % video_id,
+            ie=NexxIE.ie_key(), video_id=video_id)

From ae2384ff5f9d6a18209139a4b2e2cd33921be63a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 16:04:39 +0700
Subject: [PATCH 056/510] [tele5] Improve extraction (closes #7805, closes
 #7922, closes #17331, closes #17414)

---
 youtube_dl/extractor/tele5.py | 39 ++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/tele5.py b/youtube_dl/extractor/tele5.py
index cfa8d2475..25573e49f 100644
--- a/youtube_dl/extractor/tele5.py
+++ b/youtube_dl/extractor/tele5.py
@@ -1,38 +1,43 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from .nexx import NexxIE
+from ..compat import compat_urlparse
 
 
 class Tele5IE(InfoExtractor):
-    _VALID_URL = r'https://www\.tele5\.de/(?:mediathek/filme-online/videos\?vid=|tv/)(?P<display_id>[\w-]+)'
-
+    _VALID_URL = r'https?://(?:www\.)?tele5\.de/(?:mediathek|tv)/(?P<id>[^?#&]+)'
     _TESTS = [{
-        'url': 'https://www.tele5.de/mediathek/filme-online/videos?vid=1550589',
+        'url': 'https://www.tele5.de/mediathek/filme-online/videos?vid=1549416',
         'info_dict': {
-            'id': '1550589',
+            'id': '1549416',
             'ext': 'mp4',
-            'upload_date': '20180822',
-            'timestamp': 1534927316,
-            'title': 'SchleFaZ: Atomic Shark',
-        }
+            'upload_date': '20180814',
+            'timestamp': 1534290623,
+            'title': 'Pandorum',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.tele5.de/tv/kalkofes-mattscheibe/video-clips/politik-und-gesellschaft?ve_id=1551191',
+        'only_matching': True,
     }, {
         'url': 'https://www.tele5.de/tv/dark-matter/videos',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        video_id = (qs.get('vid') or qs.get('ve_id') or [None])[0]
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._html_search_regex(
-            r'id\s*=\s*["\']video-player["\']\s*data-id\s*=\s*["\']([0-9]+)["\']',
-            webpage, 'video_id')
+        if not video_id:
+            display_id = self._match_id(url)
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._html_search_regex(
+                r'id\s*=\s*["\']video-player["\'][^>]+data-id\s*=\s*["\'](\d+)',
+                webpage, 'video id')
 
         return self.url_result(
             'https://api.nexx.cloud/v3/759/videos/byid/%s' % video_id,

From 6f9f3340bb4c837d86067856e12232fc9695e698 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 17:24:09 +0700
Subject: [PATCH 057/510] [dtube] PEP 8 (#17455)

---
 youtube_dl/extractor/dtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dtube.py b/youtube_dl/extractor/dtube.py
index 4ca97f860..5887887e1 100644
--- a/youtube_dl/extractor/dtube.py
+++ b/youtube_dl/extractor/dtube.py
@@ -59,7 +59,7 @@ class DTubeIE(InfoExtractor):
             try:
                 self.to_screen('%s: Checking %s video format URL' % (video_id, format_id))
                 self._downloader._opener.open(video_url, timeout=5).close()
-            except timeout as e:
+            except timeout:
                 self.to_screen(
                     '%s: %s URL is invalid, skipping' % (video_id, format_id))
                 continue

From 13ef64fd9340794e214ba64c258fa0999fbd0a17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 17:24:34 +0700
Subject: [PATCH 058/510] [motherless] PEP 8 (#17455)

---
 youtube_dl/extractor/motherless.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index bed5645f2..d4bd273b6 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -167,9 +167,9 @@ class MotherlessGroupIE(InfoExtractor):
         if not entries:
             entries = [
                 self.url_result(
-                    compat_urlparse.urljoin(base, '/' + video_id),
-                    ie=MotherlessIE.ie_key(), video_id=video_id)
-                for video_id in orderedSet(re.findall(
+                    compat_urlparse.urljoin(base, '/' + entry_id),
+                    ie=MotherlessIE.ie_key(), video_id=entry_id)
+                for entry_id in orderedSet(re.findall(
                     r'data-codename=["\']([A-Z0-9]+)', webpage))]
         return entries
 

From 2c9d3b9962e28b61615e382bfcaf6db6d6987c09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 17:24:48 +0700
Subject: [PATCH 059/510] [seznamzpravy] PEP 8 (#17455)

---
 youtube_dl/extractor/seznamzpravy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/seznamzpravy.py b/youtube_dl/extractor/seznamzpravy.py
index 6d4e3b76d..7a1c7e38b 100644
--- a/youtube_dl/extractor/seznamzpravy.py
+++ b/youtube_dl/extractor/seznamzpravy.py
@@ -164,6 +164,6 @@ class SeznamZpravyArticleIE(InfoExtractor):
         description = info.get('description') or self._og_search_description(webpage)
 
         return self.playlist_result([
-            self.url_result(url, ie=SeznamZpravyIE.ie_key())
-            for url in SeznamZpravyIE._extract_urls(webpage)],
+            self.url_result(entry_url, ie=SeznamZpravyIE.ie_key())
+            for entry_url in SeznamZpravyIE._extract_urls(webpage)],
             article_id, title, description)

From 2e4350eec6981ffb12617abda7623e4fb565ce8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Sep 2018 17:24:59 +0700
Subject: [PATCH 060/510] [generic] PEP 8 (#17455)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1db154c4f..76ef01332 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3112,7 +3112,7 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 foxnews_urls, video_id, video_title, ie=FoxNewsIE.ie_key())
 
-        sharevideos_urls = [mobj.group('url') for mobj in re.finditer(
+        sharevideos_urls = [sharevideos_mobj.group('url') for sharevideos_mobj in re.finditer(
             r'<iframe[^>]+?\bsrc\s*=\s*(["\'])(?P<url>(?:https?:)?//embed\.share-videos\.se/auto/embed/\d+\?.*?\buid=\d+.*?)\1',
             webpage)]
         if sharevideos_urls:

From a2637a2dda5e90b83bd2f403d06ad1ce21fc8c5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Sep 2018 01:34:49 +0700
Subject: [PATCH 061/510] [iprima] Add support for prima.iprima.cz (closes
 #17514)

---
 youtube_dl/extractor/iprima.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 3c4b7e48b..1d58d6e85 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://(?:play|prima)\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'
     _GEO_BYPASS = False
 
     _TESTS = [{
@@ -33,6 +33,14 @@ class IPrimaIE(InfoExtractor):
         # geo restricted
         'url': 'http://play.iprima.cz/closer-nove-pripady/closer-nove-pripady-iv-1',
         'only_matching': True,
+    }, {
+        # iframe api.play-backend.iprima.cz
+        'url': 'https://prima.iprima.cz/my-little-pony/mapa-znameni-2-2',
+        'only_matching': True,
+    }, {
+        # iframe prima.iprima.cz
+        'url': 'https://prima.iprima.cz/porady/jak-se-stavi-sen/rodina-rathousova-praha',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -42,7 +50,10 @@ class IPrimaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')
+        video_id = self._search_regex(
+            (r'<iframe[^>]+\bsrc=["\'](?:https?:)?//(?:api\.play-backend\.iprima\.cz/prehravac/embedded|prima\.iprima\.cz/[^/]+/[^/]+)\?.*?\bid=(p\d+)',
+             r'data-product="([^"]+)">'),
+            webpage, 'real id')
 
         playerpage = self._download_webpage(
             'http://play.iprima.cz/prehravac/init',

From 25d110be30b92f785617140b0617a73d8eec5f7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Sep 2018 02:37:22 +0700
Subject: [PATCH 062/510] [utils] Properly recognize AV1 codec (closes #17506)

---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8da5ccc56..9e28e008f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -785,6 +785,10 @@ class TestUtil(unittest.TestCase):
             'vcodec': 'h264',
             'acodec': 'aac',
         })
+        self.assertEqual(parse_codecs('av01.0.05M.08'), {
+            'vcodec': 'av01.0.05M.08',
+            'acodec': 'none',
+        })
 
     def test_escape_rfc3986(self):
         reserved = "!*'();:@&=+$,/?#[]"
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bcfb72d43..e84d35d4d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2477,7 +2477,7 @@ def parse_codecs(codecs_str):
     vcodec, acodec = None, None
     for full_codec in splited_codecs:
         codec = full_codec.split('.')[0]
-        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1'):
+        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01'):
             if not vcodec:
                 vcodec = full_codec
         elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):

From 14f577e31c3d11e84bda5048d11bc7c31a830e2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Sep 2018 02:45:44 +0700
Subject: [PATCH 063/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ac95c9b71..c02450bf4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+version <unreleased>
+
+Core
++ [utils] Properly recognize AV1 codec (#17506)
+
+Extractors
++ [iprima] Add support for prima.iprima.cz (#17514)
++ [tele5] Add support for tele5.de (#7805, #7922, #17331, #17414)
+* [nbc] Fix extraction of percent encoded URLs (#17374)
+
+
 version 2018.09.08
 
 Extractors

From 8476b4fd91ce71c052a3bdea9e010e5196331606 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Sep 2018 02:48:37 +0700
Subject: [PATCH 064/510] release 2018.09.10

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2d67247e6..f41266f32 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.08*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.08**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.09.08
+[debug] youtube-dl version 2018.09.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c02450bf4..d184f69ee 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.09.10
 
 Core
 + [utils] Properly recognize AV1 codec (#17506)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f0d6be901..9b8601751 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -847,6 +847,7 @@
  - **techtv.mit.edu**
  - **ted**
  - **Tele13**
+ - **Tele5**
  - **TeleBruxelles**
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 716d9ffe0..b078c4993 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.09.08'
+__version__ = '2018.09.10'

From 96dbf70de63c08fc27e9bd7bec97670325225758 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Sep 2018 02:24:32 +0700
Subject: [PATCH 065/510] [eporner] Extract JSON-LD (closes #17519)

---
 youtube_dl/extractor/eporner.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 6d03d7095..c050bf9df 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -9,6 +9,7 @@ from ..utils import (
     encode_base_n,
     ExtractorError,
     int_or_none,
+    merge_dicts,
     parse_duration,
     str_to_int,
     url_or_none,
@@ -25,10 +26,16 @@ class EpornerIE(InfoExtractor):
             'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
             'ext': 'mp4',
             'title': 'Infamous Tiffany Teen Strip Tease Video',
+            'description': 'md5:764f39abf932daafa37485eb46efa152',
+            'timestamp': 1232520922,
+            'upload_date': '20090121',
             'duration': 1838,
             'view_count': int,
             'age_limit': 18,
         },
+        'params': {
+            'proxy': '127.0.0.1:8118'
+        }
     }, {
         # New (May 2016) URL layout
         'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0/Star-Wars-XXX-Parody/',
@@ -104,12 +111,15 @@ class EpornerIE(InfoExtractor):
                     })
         self._sort_formats(formats)
 
-        duration = parse_duration(self._html_search_meta('duration', webpage))
+        json_ld = self._search_json_ld(webpage, display_id, default={})
+
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, default=None))
         view_count = str_to_int(self._search_regex(
             r'id="cinemaviews">\s*([0-9,]+)\s*<small>views',
             webpage, 'view count', fatal=False))
 
-        return {
+        return merge_dicts(json_ld, {
             'id': video_id,
             'display_id': display_id,
             'title': title,
@@ -117,4 +127,4 @@ class EpornerIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
             'age_limit': 18,
-        }
+        })

From 79facb27735baf27d65baf050121d68e26654a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Sep 2018 02:29:45 +0700
Subject: [PATCH 066/510] [tube8] Fix metadata extraction (closes #17520)

---
 youtube_dl/extractor/tube8.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 368c45729..db93b0182 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -45,7 +45,7 @@ class Tube8IE(KeezMoviesIE):
                 r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
 
         description = self._html_search_regex(
-            r'>Description:</strong>\s*(.+?)\s*<', webpage, 'description', fatal=False)
+            r'(?s)Description:</dt>\s*<dd>(.+?)</dd>', webpage, 'description', fatal=False)
         uploader = self._html_search_regex(
             r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
@@ -55,19 +55,19 @@ class Tube8IE(KeezMoviesIE):
         dislike_count = int_or_none(self._search_regex(
             r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
         view_count = str_to_int(self._search_regex(
-            r'<strong>Views: </strong>([\d,\.]+)\s*</li>',
+            r'Views:\s*</dt>\s*<dd>([\d,\.]+)',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._search_regex(
             r'<span id="allCommentsCount">(\d+)</span>',
             webpage, 'comment count', fatal=False))
 
         category = self._search_regex(
-            r'Category:\s*</strong>\s*<a[^>]+href=[^>]+>([^<]+)',
+            r'Category:\s*</dt>\s*<dd>\s*<a[^>]+href=[^>]+>([^<]+)',
             webpage, 'category', fatal=False)
         categories = [category] if category else None
 
         tags_str = self._search_regex(
-            r'(?s)Tags:\s*</strong>(.+?)</(?!a)',
+            r'(?s)Tags:\s*</dt>\s*<dd>(.+?)</(?!a)',
             webpage, 'tags', fatal=False)
         tags = [t for t in re.findall(
             r'<a[^>]+href=[^>]+>([^<]+)', tags_str)] if tags_str else None

From f5b0175349aa85a1b366d12a487ebd66aec5e1b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Sep 2018 02:41:05 +0700
Subject: [PATCH 067/510] [vzaar] Add support for HLS

---
 youtube_dl/extractor/vzaar.py | 60 +++++++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/vzaar.py b/youtube_dl/extractor/vzaar.py
index 02fcd52c7..6000671c3 100644
--- a/youtube_dl/extractor/vzaar.py
+++ b/youtube_dl/extractor/vzaar.py
@@ -4,15 +4,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     float_or_none,
+    unified_timestamp,
+    url_or_none,
 )
 
 
 class VzaarIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|view)\.)?vzaar\.com/(?:videos/)?(?P<id>\d+)'
     _TESTS = [{
+        # HTTP and HLS
         'url': 'https://vzaar.com/videos/1152805',
         'md5': 'bde5ddfeb104a6c56a93a06b04901dbf',
         'info_dict': {
@@ -40,24 +44,48 @@ class VzaarIE(InfoExtractor):
         video_id = self._match_id(url)
         video_data = self._download_json(
             'http://view.vzaar.com/v2/%s/video' % video_id, video_id)
-        source_url = video_data['sourceUrl']
 
-        info = {
+        title = video_data['videoTitle']
+
+        formats = []
+
+        source_url = url_or_none(video_data.get('sourceUrl'))
+        if source_url:
+            f = {
+                'url': source_url,
+                'format_id': 'http',
+            }
+            if 'audio' in source_url:
+                f.update({
+                    'vcodec': 'none',
+                    'ext': 'mp3',
+                })
+            else:
+                f.update({
+                    'width': int_or_none(video_data.get('width')),
+                    'height': int_or_none(video_data.get('height')),
+                    'ext': 'mp4',
+                    'fps': float_or_none(video_data.get('fps')),
+                })
+            formats.append(f)
+
+        video_guid = video_data.get('guid')
+        usp = video_data.get('usp')
+        if isinstance(video_guid, compat_str) and isinstance(usp, dict):
+            m3u8_url = ('http://fable.vzaar.com/v4/usp/%s/%s.ism/.m3u8?'
+                        % (video_guid, video_id)) + '&'.join(
+                '%s=%s' % (k, v) for k, v in usp.items())
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        self._sort_formats(formats)
+
+        return {
             'id': video_id,
-            'title': video_data['videoTitle'],
-            'url': source_url,
+            'title': title,
             'thumbnail': self._proto_relative_url(video_data.get('poster')),
             'duration': float_or_none(video_data.get('videoDuration')),
+            'timestamp': unified_timestamp(video_data.get('ts')),
+            'formats': formats,
         }
-        if 'audio' in source_url:
-            info.update({
-                'vcodec': 'none',
-                'ext': 'mp3',
-            })
-        else:
-            info.update({
-                'width': int_or_none(video_data.get('width')),
-                'height': int_or_none(video_data.get('height')),
-                'ext': 'mp4',
-            })
-        return info

From db348e88495bd87adcf058bb6c0a26c1fc591f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Sep 2018 23:26:38 +0700
Subject: [PATCH 068/510] [twitch:clips] Extend _VALID_URL (closes #17559)

---
 youtube_dl/extractor/twitch.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b39972b1e..26661fdf6 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -559,7 +559,8 @@ class TwitchStreamIE(TwitchBaseIE):
                     TwitchAllVideosIE,
                     TwitchUploadsIE,
                     TwitchPastBroadcastsIE,
-                    TwitchHighlightsIE))
+                    TwitchHighlightsIE,
+                    TwitchClipsIE))
                 else super(TwitchStreamIE, cls).suitable(url))
 
     def _real_extract(self, url):
@@ -633,7 +634,7 @@ class TwitchStreamIE(TwitchBaseIE):
 
 class TwitchClipsIE(TwitchBaseIE):
     IE_NAME = 'twitch:clips'
-    _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:clips\.twitch\.tv/(?:[^/]+/)*|(?:www\.)?twitch\.tv/[^/]+/clip/)(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'https://clips.twitch.tv/FaintLightGullWholeWheat',
@@ -653,6 +654,9 @@ class TwitchClipsIE(TwitchBaseIE):
         # multiple formats
         'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
         'only_matching': True,
+    }, {
+        'url': 'https://www.twitch.tv/sergeynixon/clip/StormyThankfulSproutFutureMan',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0f2aa0dcaa5ebffbca781f8ecd15c4d52a80f011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Sep 2018 23:56:03 +0700
Subject: [PATCH 069/510] [asiancrush] Fix extraction (closes #15630)

---
 youtube_dl/extractor/asiancrush.py | 42 ++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/asiancrush.py b/youtube_dl/extractor/asiancrush.py
index 594c88c9c..6d71c5ad5 100644
--- a/youtube_dl/extractor/asiancrush.py
+++ b/youtube_dl/extractor/asiancrush.py
@@ -8,7 +8,6 @@ from .kaltura import KalturaIE
 from ..utils import (
     extract_attributes,
     remove_end,
-    urlencode_postdata,
 )
 
 
@@ -34,19 +33,40 @@ class AsianCrushIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        data = self._download_json(
-            'https://www.asiancrush.com/wp-admin/admin-ajax.php', video_id,
-            data=urlencode_postdata({
-                'postid': video_id,
-                'action': 'get_channel_kaltura_vars',
-            }))
+        webpage = self._download_webpage(url, video_id)
 
-        entry_id = data['entry_id']
+        entry_id, partner_id, title = [None] * 3
+
+        vars = self._parse_json(
+            self._search_regex(
+                r'iEmbedVars\s*=\s*({.+?})', webpage, 'embed vars',
+                default='{}'), video_id, fatal=False)
+        if vars:
+            entry_id = vars.get('entry_id')
+            partner_id = vars.get('partner_id')
+            title = vars.get('vid_label')
+
+        if not entry_id:
+            entry_id = self._search_regex(
+                r'\bentry_id["\']\s*:\s*["\'](\d+)', webpage, 'entry id')
+
+        player = self._download_webpage(
+            'https://api.asiancrush.com/embeddedVideoPlayer', video_id,
+            query={'id': entry_id})
+
+        kaltura_id = self._search_regex(
+            r'entry_id["\']\s*:\s*(["\'])(?P<id>(?:(?!\1).)+)\1', player,
+            'kaltura id', group='id')
+
+        if not partner_id:
+            partner_id = self._search_regex(
+                r'/p(?:artner_id)?/(\d+)', player, 'partner id',
+                default='513551')
 
         return self.url_result(
-            'kaltura:%s:%s' % (data['partner_id'], entry_id),
-            ie=KalturaIE.ie_key(), video_id=entry_id,
-            video_title=data.get('vid_label'))
+            'kaltura:%s:%s' % (partner_id, kaltura_id),
+            ie=KalturaIE.ie_key(), video_id=kaltura_id,
+            video_title=title)
 
 
 class AsianCrushPlaylistIE(InfoExtractor):

From 15bf2ca0da0ae94dbc6c3b76c89910ceaea88bd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 00:42:27 +0700
Subject: [PATCH 070/510] [porntube] Fix extraction (closes #17541)

---
 youtube_dl/extractor/fourtube.py | 94 ++++++++++++++++++++++++++------
 1 file changed, 76 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index ad273a0e7..20391b2bb 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,15 +3,44 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_b64decode,
+    compat_str,
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
 from ..utils import (
+    int_or_none,
     parse_duration,
     parse_iso8601,
     str_to_int,
+    try_get,
+    unified_timestamp,
+    url_or_none,
 )
 
 
 class FourTubeBaseIE(InfoExtractor):
+    _TKN_HOST = 'tkn.kodicdn.com'
+
+    def _extract_formats(self, url, video_id, media_id, sources):
+        token_url = 'https://%s/%s/desktop/%s' % (
+            self._TKN_HOST, media_id, '+'.join(sources))
+
+        parsed_url = compat_urlparse.urlparse(url)
+        tokens = self._download_json(token_url, video_id, data=b'', headers={
+            'Origin': '%s://%s' % (parsed_url.scheme, parsed_url.hostname),
+            'Referer': url,
+        })
+        formats = [{
+            'url': tokens[format]['token'],
+            'format_id': format + 'p',
+            'resolution': format + 'p',
+            'quality': int(format),
+        } for format in sources]
+        self._sort_formats(formats)
+        return formats
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         kind, video_id, display_id = mobj.group('kind', 'id', 'display_id')
@@ -68,21 +97,7 @@ class FourTubeBaseIE(InfoExtractor):
             media_id = params[0]
             sources = ['%s' % p for p in params[2]]
 
-        token_url = 'https://tkn.kodicdn.com/{0}/desktop/{1}'.format(
-            media_id, '+'.join(sources))
-
-        parsed_url = compat_urlparse.urlparse(url)
-        tokens = self._download_json(token_url, video_id, data=b'', headers={
-            'Origin': '%s://%s' % (parsed_url.scheme, parsed_url.hostname),
-            'Referer': url,
-        })
-        formats = [{
-            'url': tokens[format]['token'],
-            'format_id': format + 'p',
-            'resolution': format + 'p',
-            'quality': int(format),
-        } for format in sources]
-        self._sort_formats(formats)
+        formats = self._extract_formats(url, video_id, media_id, sources)
 
         return {
             'id': video_id,
@@ -164,6 +179,7 @@ class FuxIE(FourTubeBaseIE):
 class PornTubeIE(FourTubeBaseIE):
     _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?porntube\.com/(?:videos/(?P<display_id>[^/]+)_|embed/)(?P<id>\d+)'
     _URL_TEMPLATE = 'https://www.porntube.com/videos/video_%s'
+    _TKN_HOST = 'tkn.porntube.com'
     _TESTS = [{
         'url': 'https://www.porntube.com/videos/teen-couple-doing-anal_7089759',
         'info_dict': {
@@ -171,13 +187,12 @@ class PornTubeIE(FourTubeBaseIE):
             'ext': 'mp4',
             'title': 'Teen couple doing anal',
             'uploader': 'Alexy',
-            'uploader_id': 'Alexy',
+            'uploader_id': '91488',
             'upload_date': '20150606',
             'timestamp': 1433595647,
             'duration': 5052,
             'view_count': int,
             'like_count': int,
-            'categories': list,
             'age_limit': 18,
         },
         'params': {
@@ -191,6 +206,49 @@ class PornTubeIE(FourTubeBaseIE):
         'only_matching': True,
     }]
 
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id, display_id = mobj.group('id', 'display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video = self._parse_json(
+            self._search_regex(
+                r'INITIALSTATE\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
+                webpage, 'data', group='value'), video_id,
+            transform_source=lambda x: compat_urllib_parse_unquote(
+                compat_b64decode(x).decode('utf-8')))['page']['video']
+
+        title = video['title']
+        media_id = video['mediaId']
+        sources = [compat_str(e['height'])
+                   for e in video['encodings'] if e.get('height')]
+        formats = self._extract_formats(url, video_id, media_id, sources)
+
+        thumbnail = url_or_none(video.get('masterThumb'))
+        uploader = try_get(video, lambda x: x['user']['username'], compat_str)
+        uploader_id = compat_str(try_get(video, lambda x: x['user']['id'], int))
+        like_count = int_or_none(video.get('likes'))
+        dislike_count = int_or_none(video.get('dislikes'))
+        view_count = int_or_none(video.get('playsQty'))
+        duration = int_or_none(video.get('durationInSeconds'))
+        timestamp = unified_timestamp(video.get('publishedAt'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'timestamp': timestamp,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'view_count': view_count,
+            'duration': duration,
+            'age_limit': 18,
+        }
+
 
 class PornerBrosIE(FourTubeBaseIE):
     _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?pornerbros\.com/(?:videos/(?P<display_id>[^/]+)_|embed/)(?P<id>\d+)'

From 6f1f59f39cf934d5ccae3ca2e6fd4eaab726137f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 01:23:36 +0700
Subject: [PATCH 071/510] [extractor/common] Introduce channel meta fields

---
 youtube_dl/extractor/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b8bbaf81a..8eab5947f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -211,6 +211,11 @@ class InfoExtractor(object):
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
     uploader_url:   Full URL to a personal webpage of the video uploader.
+    channel:        Full name of the channel the video is uploaded on.
+                    Note that channel fields may or may noy repeat uploader
+                    fields. This depends on a particular extractor.
+    channel_id:     Id of the channel.
+    channel_url:    Full URL to a channel webpage.
     location:       Physical location where the video was filmed.
     subtitles:      The available subtitles as a dictionary in the format
                     {tag: subformats}. "tag" is usually a language code, and

From dd4c449219976ad6e80fef74a3431c5553cf5031 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 01:24:26 +0700
Subject: [PATCH 072/510] [youtube] Extract channel meta fields (closes #9676,
 closes #12939)

---
 youtube_dl/extractor/youtube.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 27047425d..2fe074cb4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -490,6 +490,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
+                'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
+                'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
                 'upload_date': '20121002',
                 'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
@@ -1907,6 +1909,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
+        channel_id = self._html_search_meta(
+            'channelId', video_webpage, 'channel id')
+        channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
+
         # thumbnail image
         # We try first to get a high quality image:
         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
@@ -2078,6 +2084,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
             'uploader_url': video_uploader_url,
+            'channel_id': channel_id,
+            'channel_url': channel_url,
             'upload_date': upload_date,
             'license': video_license,
             'creator': video_creator or artist,

From d03beddf0f0b460d1b58c98173e2d4620d1d11b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 01:24:48 +0700
Subject: [PATCH 073/510] [vimeo] Extract channel meta fields

---
 youtube_dl/extractor/vimeo.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index e49b233f2..95d368cc1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -299,10 +299,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/atencio',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
+                'channel_id': 'keypeele',
+                'channel_url': r're:https?://(?:www\.)?vimeo\.com/channels/keypeele',
                 'timestamp': 1380339469,
                 'upload_date': '20130928',
                 'duration': 187,
             },
+            'expected_warnings': ['Unable to download JSON metadata'],
         },
         {
             'url': 'http://vimeo.com/76979871',
@@ -355,11 +358,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/channels/tributes/6213729',
             'info_dict': {
                 'id': '6213729',
-                'ext': 'mov',
+                'ext': 'mp4',
                 'title': 'Vimeo Tribute: The Shining',
                 'uploader': 'Casey Donahue',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/caseydonahue',
                 'uploader_id': 'caseydonahue',
+                'channel_url': r're:https?://(?:www\.)?vimeo\.com/channels/tributes',
+                'channel_id': 'tributes',
                 'timestamp': 1250886430,
                 'upload_date': '20090821',
                 'description': 'md5:bdbf314014e58713e6e5b66eb252f4a6',
@@ -465,6 +470,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if 'Referer' not in headers:
             headers['Referer'] = url
 
+        channel_id = self._search_regex(
+            r'vimeo\.com/channels/([^/]+)', url, 'channel id', default=None)
+
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -652,6 +660,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 r'<link[^>]+rel=["\']license["\'][^>]+href=(["\'])(?P<license>(?:(?!\1).)+)\1',
                 webpage, 'license', default=None, group='license')
 
+        channel_url = 'https://vimeo.com/channels/%s' % channel_id if channel_id else None
+
         info_dict = {
             'id': video_id,
             'formats': formats,
@@ -662,6 +672,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'like_count': like_count,
             'comment_count': comment_count,
             'license': cc_license,
+            'channel_id': channel_id,
+            'channel_url': channel_url,
         }
 
         info_dict = merge_dicts(info_dict, info_dict_config, json_ld)

From 127103b643c5105bd361c5e3be267ac40843ccdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 01:25:32 +0700
Subject: [PATCH 074/510] [porntube] Extract channel meta fields

---
 youtube_dl/extractor/fourtube.py | 33 +++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 20391b2bb..a9a1f911e 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -13,6 +13,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    str_or_none,
     str_to_int,
     try_get,
     unified_timestamp,
@@ -198,6 +199,26 @@ class PornTubeIE(FourTubeBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.porntube.com/videos/squirting-teen-ballerina-ecg_1331406',
+        'info_dict': {
+            'id': '1331406',
+            'ext': 'mp4',
+            'title': 'Squirting Teen Ballerina on ECG',
+            'uploader': 'Exploited College Girls',
+            'uploader_id': '665',
+            'channel': 'Exploited College Girls',
+            'channel_id': '665',
+            'upload_date': '20130920',
+            'timestamp': 1379685485,
+            'duration': 851,
+            'view_count': int,
+            'like_count': int,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.porntube.com/embed/7089759',
         'only_matching': True,
@@ -227,7 +248,11 @@ class PornTubeIE(FourTubeBaseIE):
 
         thumbnail = url_or_none(video.get('masterThumb'))
         uploader = try_get(video, lambda x: x['user']['username'], compat_str)
-        uploader_id = compat_str(try_get(video, lambda x: x['user']['id'], int))
+        uploader_id = str_or_none(try_get(
+            video, lambda x: x['user']['id'], int))
+        channel = try_get(video, lambda x: x['channel']['name'], compat_str)
+        channel_id = str_or_none(try_get(
+            video, lambda x: x['channel']['id'], int))
         like_count = int_or_none(video.get('likes'))
         dislike_count = int_or_none(video.get('dislikes'))
         view_count = int_or_none(video.get('playsQty'))
@@ -239,8 +264,10 @@ class PornTubeIE(FourTubeBaseIE):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
+            'uploader': uploader or channel,
+            'uploader_id': uploader_id or channel_id,
+            'channel': channel,
+            'channel_id': channel_id,
             'timestamp': timestamp,
             'like_count': like_count,
             'dislike_count': dislike_count,

From 0e7b8d3eaca710fbae8aa5531e28a6b6f05b387a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 01:53:01 +0700
Subject: [PATCH 075/510] [extractor/common] Fix typos

---
 youtube_dl/extractor/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8eab5947f..2dbf81e6e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -212,7 +212,7 @@ class InfoExtractor(object):
     uploader_id:    Nickname or id of the video uploader.
     uploader_url:   Full URL to a personal webpage of the video uploader.
     channel:        Full name of the channel the video is uploaded on.
-                    Note that channel fields may or may noy repeat uploader
+                    Note that channel fields may or may not repeat uploader
                     fields. This depends on a particular extractor.
     channel_id:     Id of the channel.
     channel_url:    Full URL to a channel webpage.
@@ -1706,9 +1706,9 @@ class InfoExtractor(object):
                 # However, this is not always respected, for example, [2]
                 # contains EXT-X-STREAM-INF tag which references AUDIO
                 # rendition group but does not have CODECS and despite
-                # referencing audio group an audio group, it represents
-                # a complete (with audio and video) format. So, for such cases
-                # we will ignore references to rendition groups and treat them
+                # referencing an audio group it represents a complete
+                # (with audio and video) format. So, for such cases we will
+                # ignore references to rendition groups and treat them
                 # as complete formats.
                 if audio_group_id and codecs and f.get('vcodec') != 'none':
                     audio_group = groups.get(audio_group_id)

From 3661ebf2b6e0cb951dca5b4498a1d6885cbf7b56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Sep 2018 02:04:43 +0700
Subject: [PATCH 076/510] [pornhub] Extract upload date (closes #17574)

---
 youtube_dl/extractor/pornhub.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 6782848d9..19eaf389f 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -40,6 +40,7 @@ class PornHubIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
             'uploader': 'Babes',
+            'upload_date': '20130628',
             'duration': 361,
             'view_count': int,
             'like_count': int,
@@ -57,6 +58,7 @@ class PornHubIE(InfoExtractor):
             'ext': 'mp4',
             'title': '重庆婷婷女王足交',
             'uploader': 'Unknown',
+            'upload_date': '20150213',
             'duration': 1753,
             'view_count': int,
             'like_count': int,
@@ -237,8 +239,14 @@ class PornHubIE(InfoExtractor):
                 video_urls.append((video_url, None))
                 video_urls_set.add(video_url)
 
+        upload_date = None
         formats = []
         for video_url, height in video_urls:
+            if not upload_date:
+                upload_date = self._search_regex(
+                    r'/(\d{6}/\d{2})/', video_url, 'upload data', default=None)
+                if upload_date:
+                    upload_date = upload_date.replace('/', '')
             tbr = None
             mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
             if mobj:
@@ -278,6 +286,7 @@ class PornHubIE(InfoExtractor):
         return {
             'id': video_id,
             'uploader': video_uploader,
+            'upload_date': upload_date,
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,

From 8b40c92724453995701b75559b190075d6dbfd7d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 15 Sep 2018 06:30:57 +0100
Subject: [PATCH 077/510] [vimeo] redirect to feature url only in the case of a
 trailer(closes #14591)

---
 youtube_dl/extractor/vimeo.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 95d368cc1..0a9239b62 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -571,19 +571,23 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if config.get('view') == 4:
                 config = self._verify_player_video_password(redirect_url, video_id)
 
+        vod = config.get('video', {}).get('vod', {})
+
         def is_rented():
             if '>You rented this title.<' in webpage:
                 return True
             if config.get('user', {}).get('purchased'):
                 return True
-            label = try_get(
-                config, lambda x: x['video']['vod']['purchase_options'][0]['label_string'], compat_str)
-            if label and label.startswith('You rented this'):
-                return True
+            for purchase_option in vod.get('purchase_options', []):
+                if purchase_option.get('purchased'):
+                    return True
+                label = purchase_option.get('label_string')
+                if label and (label.startswith('You rented this') or label.endswith(' remaining')):
+                    return True
             return False
 
-        if is_rented():
-            feature_id = config.get('video', {}).get('vod', {}).get('feature_id')
+        if is_rented() and vod.get('is_trailer'):
+            feature_id = vod.get('feature_id')
             if feature_id and not data.get('force_feature_id', False):
                 return self.url_result(smuggle_url(
                     'https://player.vimeo.com/player/%s' % feature_id,

From d9b0d118adfa365e8ed67de277f5e83e29c4635c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Sep 2018 23:52:27 +0700
Subject: [PATCH 078/510] [vrv] Make format ids deterministic

---
 youtube_dl/extractor/vrv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 921e9e172..ac0819c7c 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -90,7 +90,13 @@ class VRVIE(VRVBaseIE):
     def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang):
         if not url or stream_format not in ('hls', 'dash'):
             return []
-        stream_id = hardsub_lang or audio_lang
+        assert audio_lang or hardsub_lang
+        stream_id_list = []
+        if audio_lang:
+            stream_id_list.append('audio-%s' % audio_lang)
+        if hardsub_lang:
+            stream_id_list.append('hardsub-%s' % hardsub_lang)
+        stream_id = '-'.join(stream_id_list)
         format_id = '%s-%s' % (stream_format, stream_id)
         if stream_format == 'hls':
             adaptive_formats = self._extract_m3u8_formats(

From 1084563eaab3a039ea8ccb65f2507e919d8e11f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Sep 2018 23:54:25 +0700
Subject: [PATCH 079/510] [crunchyroll] Prefer hardsubless formats and formats
 in locale language

---
 youtube_dl/extractor/crunchyroll.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index ba8b9fa7e..af786d096 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -445,6 +445,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             webpage, 'vilos media', default='{}'), video_id)
         media_metadata = media.get('metadata') or {}
 
+        language = self._search_regex(
+            r'(?:vilos\.config\.player\.language|LOCALE)\s*=\s*(["\'])(?P<lang>(?:(?!\1).)+)\1',
+            webpage, 'language', default=None, group='lang')
+
         video_title = self._html_search_regex(
             r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
             webpage, 'video_title')
@@ -466,9 +470,22 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         formats = []
         for stream in media.get('streams', []):
-            formats.extend(self._extract_vrv_formats(
+            audio_lang = stream.get('audio_lang')
+            hardsub_lang = stream.get('hardsub_lang')
+            vrv_formats = self._extract_vrv_formats(
                 stream.get('url'), video_id, stream.get('format'),
-                stream.get('audio_lang'), stream.get('hardsub_lang')))
+                audio_lang, hardsub_lang)
+            for f in vrv_formats:
+                if not hardsub_lang:
+                    f['preference'] = 1
+                language_preference = 0
+                if audio_lang == language:
+                    language_preference += 1
+                if hardsub_lang == language:
+                    language_preference += 1
+                if language_preference:
+                    f['language_preference'] = language_preference
+            formats.extend(vrv_formats)
         if not formats:
             available_fmts = []
             for a, fmt in re.findall(r'(<a[^>]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage):
@@ -557,7 +574,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                         'ext': 'flv',
                     })
                     formats.append(format_info)
-        self._sort_formats(formats, ('height', 'width', 'tbr', 'fps'))
+        self._sort_formats(formats, ('preference', 'language_preference', 'height', 'width', 'tbr', 'fps'))
 
         metadata = self._call_rpc_api(
             'VideoPlayer_GetMediaMetadata', video_id,

From c11485162bcbf6f517fd9225850a048fc5f7cec1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Sep 2018 22:13:39 +0700
Subject: [PATCH 080/510] [youtube] Don't pollute default query dict (closes
 #17593)

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2fe074cb4..e80e36f98 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -259,7 +259,9 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         return True
 
     def _download_webpage_handle(self, *args, **kwargs):
-        kwargs.setdefault('query', {})['disable_polymer'] = 'true'
+        query = kwargs.get('query', {}).copy()
+        query['disable_polymer'] = 'true'
+        kwargs['query'] = query
         return super(YoutubeBaseInfoExtractor, self)._download_webpage_handle(
             *args, **compat_kwargs(kwargs))
 

From e2f61598be49b75f16b501f7a4d24f4c8c230c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Sep 2018 22:14:28 +0700
Subject: [PATCH 081/510] [twitch] Don't pollute default headers dict

---
 youtube_dl/extractor/twitch.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 26661fdf6..401615683 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -51,7 +51,9 @@ class TwitchBaseIE(InfoExtractor):
                 expected=True)
 
     def _call_api(self, path, item_id, *args, **kwargs):
-        kwargs.setdefault('headers', {})['Client-ID'] = self._CLIENT_ID
+        headers = kwargs.get('headers', {}).copy()
+        headers['Client-ID'] = self._CLIENT_ID
+        kwargs['headers'] = headers
         response = self._download_json(
             '%s/%s' % (self._API_BASE, path), item_id,
             *args, **compat_kwargs(kwargs))

From c8f6ab8c38f8397b06028cdea2839d6e25e2c809 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Sep 2018 22:14:53 +0700
Subject: [PATCH 082/510] [udemy] Don't pollute default headers dict

---
 youtube_dl/extractor/udemy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 79c45f80e..105826e9b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -122,7 +122,9 @@ class UdemyIE(InfoExtractor):
             raise ExtractorError(error_str, expected=True)
 
     def _download_webpage_handle(self, *args, **kwargs):
-        kwargs.setdefault('headers', {})['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4'
+        headers = kwargs.get('headers', {}).copy()
+        headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4'
+        kwargs['headers'] = headers
         return super(UdemyIE, self)._download_webpage_handle(
             *args, **compat_kwargs(kwargs))
 

From e504b0907082512d5252643d7fd8c5ca67225e3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Sep 2018 22:15:27 +0700
Subject: [PATCH 083/510] [adobepass] Don't pollute default headers dict

---
 youtube_dl/extractor/adobepass.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index b83b51efb..1cf2dcbf3 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -1325,8 +1325,8 @@ class AdobePassIE(InfoExtractor):
     _DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page'
 
     def _download_webpage_handle(self, *args, **kwargs):
-        headers = kwargs.get('headers', {})
-        headers.update(self.geo_verification_headers())
+        headers = self.geo_verification_headers()
+        headers.update(kwargs.get('headers', {}))
         kwargs['headers'] = headers
         return super(AdobePassIE, self)._download_webpage_handle(
             *args, **compat_kwargs(kwargs))

From d9b1cec1712c39d7e98a4ca44051643eebef8b30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Sep 2018 01:44:55 +0700
Subject: [PATCH 084/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d184f69ee..b2b2d90a1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
++ [extractor/common] Introduce channel meta fields
+
+Extractors
+* [adobepass] Don't pollute default headers dict
+* [udemy] Don't pollute default headers dict
+* [twitch] Don't pollute default headers dict
+* [youtube] Don't pollute default query dict (#17593)
+* [crunchyroll] Prefer hardsubless formats and formats in locale language
+* [vrv] Make format ids deterministic
+* [vimeo] Fix ondemand playlist extraction (#14591)
++ [pornhub] Extract upload date (#17574)
++ [porntube] Extract channel meta fields
++ [vimeo] Extract channel meta fields
++ [youtube] Extract channel meta fields (#9676, #12939)
+* [porntube] Fix extraction (#17541)
+* [asiancrush] Fix extraction (#15630)
++ [twitch:clips] Extend URL regular expression (closes #17559)
++ [vzaar] Add support for HLS
+* [tube8] Fix metadata extraction (#17520)
+* [eporner] Extract JSON-LD (#17519)
+
+
 version 2018.09.10
 
 Core

From 3a9c928426258e51c74515b09f7c6f565d60efe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Sep 2018 01:46:36 +0700
Subject: [PATCH 085/510] release 2018.09.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f41266f32..a4602287a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.09.10
+[debug] youtube-dl version 2018.09.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index b2b2d90a1..800ece790 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.09.18
 
 Core
 + [extractor/common] Introduce channel meta fields
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b078c4993..2b3b584a4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.09.10'
+__version__ = '2018.09.18'

From 28fcb7b06178cebfc361f0d1c491ea9507b0d75e Mon Sep 17 00:00:00 2001
From: Leonardo Taccari <iamleot@gmail.com>
Date: Wed, 19 Sep 2018 04:48:39 +0200
Subject: [PATCH 086/510] [raiplay:playlist] Remove a debug leftover print()

---
 youtube_dl/extractor/rai.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index f916b2619..548a6553b 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -274,7 +274,6 @@ class RaiPlayPlaylistIE(InfoExtractor):
             ('programma', 'nomeProgramma'), webpage, 'title')
         description = unescapeHTML(self._html_search_meta(
             ('description', 'og:description'), webpage, 'description'))
-        print(description)
 
         entries = []
         for mobj in re.finditer(

From 4ac73fc170f41ccb2d468a0b8563b2ec39d9e9b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Sep 2018 22:16:43 +0700
Subject: [PATCH 087/510] [popcorntv] Remove debug output

---
 youtube_dl/extractor/popcorntv.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/popcorntv.py b/youtube_dl/extractor/popcorntv.py
index ac901f426..9f834fb6c 100644
--- a/youtube_dl/extractor/popcorntv.py
+++ b/youtube_dl/extractor/popcorntv.py
@@ -58,8 +58,6 @@ class PopcornTVIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         timestamp = unified_timestamp(self._html_search_meta(
             'uploadDate', webpage, 'timestamp'))
-        print(self._html_search_meta(
-            'duration', webpage))
         duration = int_or_none(self._html_search_meta(
             'duration', webpage), invscale=60)
         view_count = int_or_none(self._html_search_meta(

From 21160a179256cfbb859948138905ffe67cb970a8 Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Sun, 23 Sep 2018 16:34:47 +0200
Subject: [PATCH 088/510] [zattoo] Fix extraction (closes #17175)

---
 youtube_dl/extractor/zattoo.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/zattoo.py b/youtube_dl/extractor/zattoo.py
index fb167c198..9c9024799 100644
--- a/youtube_dl/extractor/zattoo.py
+++ b/youtube_dl/extractor/zattoo.py
@@ -93,28 +93,30 @@ class ZattooBaseIE(InfoExtractor):
 
     def _extract_cid_and_video_info(self, video_id):
         data = self._download_json(
-            '%s/zapi/program/details' % self._HOST_URL,
+            '%s/zapi/v2/cached/program/power_details/%s' % (
+                self._HOST_URL, self._power_guide_hash),
             video_id,
             'Downloading video information',
             query={
-                'program_id': video_id,
-                'complete': True
+                'program_ids': video_id,
+                'complete': True,
             })
 
-        p = data['program']
+        p = data['programs'][0]
         cid = p['cid']
 
         info_dict = {
             'id': video_id,
-            'title': p.get('title') or p['episode_title'],
-            'description': p.get('description'),
-            'thumbnail': p.get('image_url'),
+            'title': p.get('t') or p['et'],
+            'description': p.get('d'),
+            'thumbnail': p.get('i_url'),
             'creator': p.get('channel_name'),
-            'episode': p.get('episode_title'),
-            'episode_number': int_or_none(p.get('episode_number')),
-            'season_number': int_or_none(p.get('season_number')),
+            'episode': p.get('et'),
+            'episode_number': int_or_none(p.get('e_no')),
+            'season_number': int_or_none(p.get('s_no')),
             'release_year': int_or_none(p.get('year')),
-            'categories': try_get(p, lambda x: x['categories'], list),
+            'categories': try_get(p, lambda x: x['c'], list),
+            'tags': try_get(p, lambda x: x['g'], list)
         }
 
         return cid, info_dict

From f6d7f7b474075955afb02c373b8d2a442faaa0a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Sep 2018 23:30:18 +0700
Subject: [PATCH 089/510] [zattoo] Add support for more zattoo platform sites

---
 youtube_dl/extractor/extractors.py |  12 ++
 youtube_dl/extractor/zattoo.py     | 178 ++++++++++++++++++++++++++---
 2 files changed, 172 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7dc569724..464c8d690 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1455,8 +1455,20 @@ from .youtube import (
 from .zapiks import ZapiksIE
 from .zaq1 import Zaq1IE
 from .zattoo import (
+    BBVTVIE,
+    EinsUndEinsTVIE,
+    EWETVIE,
+    GlattvisionTVIE,
+    MNetTVIE,
+    MyVisionTVIE,
+    NetPlusIE,
+    OsnatelTVIE,
+    QuantumTVIE,
     QuicklineIE,
     QuicklineLiveIE,
+    SAKTVIE,
+    VTXTVIE,
+    WalyTVIE,
     ZattooIE,
     ZattooLiveIE,
 )
diff --git a/youtube_dl/extractor/zattoo.py b/youtube_dl/extractor/zattoo.py
index 9c9024799..bbe0aecb6 100644
--- a/youtube_dl/extractor/zattoo.py
+++ b/youtube_dl/extractor/zattoo.py
@@ -18,12 +18,12 @@ from ..utils import (
 )
 
 
-class ZattooBaseIE(InfoExtractor):
-    _NETRC_MACHINE = 'zattoo'
-    _HOST_URL = 'https://zattoo.com'
-
+class ZattooPlatformBaseIE(InfoExtractor):
     _power_guide_hash = None
 
+    def _host_url(self):
+        return 'https://%s' % self._HOST
+
     def _login(self):
         username, password = self._get_login_info()
         if not username or not password:
@@ -33,13 +33,13 @@ class ZattooBaseIE(InfoExtractor):
 
         try:
             data = self._download_json(
-                '%s/zapi/v2/account/login' % self._HOST_URL, None, 'Logging in',
+                '%s/zapi/v2/account/login' % self._host_url(), None, 'Logging in',
                 data=urlencode_postdata({
                     'login': username,
                     'password': password,
                     'remember': 'true',
                 }), headers={
-                    'Referer': '%s/login' % self._HOST_URL,
+                    'Referer': '%s/login' % self._host_url(),
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                 })
         except ExtractorError as e:
@@ -53,7 +53,7 @@ class ZattooBaseIE(InfoExtractor):
 
     def _real_initialize(self):
         webpage = self._download_webpage(
-            self._HOST_URL, None, 'Downloading app token')
+            self._host_url(), None, 'Downloading app token')
         app_token = self._html_search_regex(
             r'appToken\s*=\s*(["\'])(?P<token>(?:(?!\1).)+?)\1',
             webpage, 'app token', group='token')
@@ -62,7 +62,7 @@ class ZattooBaseIE(InfoExtractor):
 
         # Will setup appropriate cookies
         self._request_webpage(
-            '%s/zapi/v2/session/hello' % self._HOST_URL, None,
+            '%s/zapi/v2/session/hello' % self._host_url(), None,
             'Opening session', data=urlencode_postdata({
                 'client_app_token': app_token,
                 'uuid': compat_str(uuid4()),
@@ -75,7 +75,7 @@ class ZattooBaseIE(InfoExtractor):
 
     def _extract_cid(self, video_id, channel_name):
         channel_groups = self._download_json(
-            '%s/zapi/v2/cached/channels/%s' % (self._HOST_URL,
+            '%s/zapi/v2/cached/channels/%s' % (self._host_url(),
                                                self._power_guide_hash),
             video_id, 'Downloading channel list',
             query={'details': False})['channel_groups']
@@ -94,7 +94,7 @@ class ZattooBaseIE(InfoExtractor):
     def _extract_cid_and_video_info(self, video_id):
         data = self._download_json(
             '%s/zapi/v2/cached/program/power_details/%s' % (
-                self._HOST_URL, self._power_guide_hash),
+                self._host_url(), self._power_guide_hash),
             video_id,
             'Downloading video information',
             query={
@@ -128,11 +128,11 @@ class ZattooBaseIE(InfoExtractor):
 
         if is_live:
             postdata_common.update({'timeshift': 10800})
-            url = '%s/zapi/watch/live/%s' % (self._HOST_URL, cid)
+            url = '%s/zapi/watch/live/%s' % (self._host_url(), cid)
         elif record_id:
-            url = '%s/zapi/watch/recording/%s' % (self._HOST_URL, record_id)
+            url = '%s/zapi/watch/recording/%s' % (self._host_url(), record_id)
         else:
-            url = '%s/zapi/watch/recall/%s/%s' % (self._HOST_URL, cid, video_id)
+            url = '%s/zapi/watch/recall/%s/%s' % (self._host_url(), cid, video_id)
 
         formats = []
         for stream_type in ('dash', 'hls', 'hls5', 'hds'):
@@ -203,13 +203,13 @@ class ZattooBaseIE(InfoExtractor):
         return info_dict
 
 
-class QuicklineBaseIE(ZattooBaseIE):
+class QuicklineBaseIE(ZattooPlatformBaseIE):
     _NETRC_MACHINE = 'quickline'
-    _HOST_URL = 'https://mobiltv.quickline.com'
+    _HOST = 'mobiltv.quickline.com'
 
 
 class QuicklineIE(QuicklineBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?mobiltv\.quickline\.com/watch/(?P<channel>[^/]+)/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?%s/watch/(?P<channel>[^/]+)/(?P<id>[0-9]+)' % re.escape(QuicklineBaseIE._HOST)
 
     _TEST = {
         'url': 'https://mobiltv.quickline.com/watch/prosieben/130671867-maze-runner-die-auserwaehlten-in-der-brandwueste',
@@ -222,7 +222,7 @@ class QuicklineIE(QuicklineBaseIE):
 
 
 class QuicklineLiveIE(QuicklineBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?mobiltv\.quickline\.com/watch/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?%s/watch/(?P<id>[^/]+)' % re.escape(QuicklineBaseIE._HOST)
 
     _TEST = {
         'url': 'https://mobiltv.quickline.com/watch/srf1',
@@ -238,8 +238,18 @@ class QuicklineLiveIE(QuicklineBaseIE):
         return self._extract_video(channel_name, video_id, is_live=True)
 
 
+class ZattooBaseIE(ZattooPlatformBaseIE):
+    _NETRC_MACHINE = 'zattoo'
+    _HOST = 'zattoo.com'
+
+
+def _make_valid_url(tmpl, host):
+    return tmpl % re.escape(host)
+
+
 class ZattooIE(ZattooBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?zattoo\.com/watch/(?P<channel>[^/]+?)/(?P<id>[0-9]+)[^/]+(?:/(?P<recid>[0-9]+))?'
+    _VALID_URL_TEMPLATE = r'https?://(?:www\.)?%s/watch/(?P<channel>[^/]+?)/(?P<id>[0-9]+)[^/]+(?:/(?P<recid>[0-9]+))?'
+    _VALID_URL = _make_valid_url(_VALID_URL_TEMPLATE, ZattooBaseIE._HOST)
 
     # Since regular videos are only available for 7 days and recorded videos
     # are only available for a specific user, we cannot have detailed tests.
@@ -271,3 +281,135 @@ class ZattooLiveIE(ZattooBaseIE):
     def _real_extract(self, url):
         channel_name = video_id = self._match_id(url)
         return self._extract_video(channel_name, video_id, is_live=True)
+
+
+class NetPlusIE(ZattooIE):
+    _NETRC_MACHINE = 'netplus'
+    _HOST = 'netplus.tv'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.netplus.tv/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class MNetTVIE(ZattooIE):
+    _NETRC_MACHINE = 'mnettv'
+    _HOST = 'tvplus.m-net.de'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.tvplus.m-net.de/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class WalyTVIE(ZattooIE):
+    _NETRC_MACHINE = 'walytv'
+    _HOST = 'player.waly.tv'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.player.waly.tv/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class BBVTVIE(ZattooIE):
+    _NETRC_MACHINE = 'bbvtv'
+    _HOST = 'bbv-tv.net'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.bbv-tv.net/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class VTXTVIE(ZattooIE):
+    _NETRC_MACHINE = 'vtxtv'
+    _HOST = 'vtxtv.ch'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.vtxtv.ch/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class MyVisionTVIE(ZattooIE):
+    _NETRC_MACHINE = 'myvisiontv'
+    _HOST = 'myvisiontv.ch'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.myvisiontv.ch/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class GlattvisionTVIE(ZattooIE):
+    _NETRC_MACHINE = 'glattvisiontv'
+    _HOST = 'iptv.glattvision.ch'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.iptv.glattvision.ch/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class SAKTVIE(ZattooIE):
+    _NETRC_MACHINE = 'saktv'
+    _HOST = 'saktv.ch'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.saktv.ch/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class EWETVIE(ZattooIE):
+    _NETRC_MACHINE = 'ewetv'
+    _HOST = 'tvonline.ewe.de'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.tvonline.ewe.de/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class QuantumTVIE(ZattooIE):
+    _NETRC_MACHINE = 'quantumtv'
+    _HOST = 'quantum-tv.com'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.quantum-tv.com/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class OsnatelTVIE(ZattooIE):
+    _NETRC_MACHINE = 'osnateltv'
+    _HOST = 'onlinetv.osnatel.de'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.onlinetv.osnatel.de/watch/abc/123-abc',
+        'only_matching': True,
+    }]
+
+
+class EinsUndEinsTVIE(ZattooIE):
+    _NETRC_MACHINE = '1und1tv'
+    _HOST = '1und1.tv'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://www.1und1.tv/watch/abc/123-abc',
+        'only_matching': True,
+    }]

From cd5a74a28e7102e877d00b1e3cffba82d28b2f2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Sep 2018 00:14:49 +0700
Subject: [PATCH 090/510] [youtube] Add support for invidio.us (closes #17613)

---
 youtube_dl/extractor/youtube.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e80e36f98..78203ef84 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -349,6 +349,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             (?:www\.)?hooktube\.com/|
                             (?:www\.)?yourepeat\.com/|
                             tube\.majestyc\.net/|
+                            (?:www\.)?invidio\.us/|
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:
@@ -1068,6 +1069,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
             'only_matching': True,
         },
+        {
+            'url': 'https://invidio.us/watch?v=BaW_jenozKc',
+            'only_matching': True,
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -2419,7 +2424,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
 
 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com channels'
-    _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
+    _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com|(?:www\.)?invidio\.us)/channel/(?P<id>[0-9A-Za-z_-]+)'
     _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
     _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
     IE_NAME = 'youtube:channel'
@@ -2440,6 +2445,9 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'UUs0ifCMCm1icqRbqhUINa0w',
             'title': 'Uploads from Deus Ex',
         },
+    }, {
+        'url': 'https://invidio.us/channel/UC23qupoDRn9YOAVzeoxjOQA',
+        'only_matching': True,
     }]
 
     @classmethod

From 60ce0c67fd1ef71463af2c036bbabf06ec26bd98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Sep 2018 23:43:41 +0700
Subject: [PATCH 091/510] [README.md] Document channel meta fields for output
 template

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index dd068a462..fdd115c9b 100644
--- a/README.md
+++ b/README.md
@@ -511,6 +511,8 @@ The basic usage is not to set any template arguments when downloading a single f
  - `timestamp` (numeric): UNIX timestamp of the moment the video became available
  - `upload_date` (string): Video upload date (YYYYMMDD)
  - `uploader_id` (string): Nickname or id of the video uploader
+ - `channel` (string): Full name of the channel the video is uploaded on
+ - `channel_id` (string): Id of the channel
  - `location` (string): Physical location where the video was filmed
  - `duration` (numeric): Length of the video in seconds
  - `view_count` (numeric): How many users have watched the video on the platform

From 8fd12a083131550476fb771c180a0734794d0b9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Sep 2018 05:38:41 +0700
Subject: [PATCH 092/510] [mediaset] Improve embed support (closes #17668)

---
 youtube_dl/extractor/generic.py  |  2 +-
 youtube_dl/extractor/mediaset.py | 38 +++++++++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 76ef01332..2a48667f0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3023,7 +3023,7 @@ class GenericIE(InfoExtractor):
                 wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
 
         # Look for Mediaset embeds
-        mediaset_urls = MediasetIE._extract_urls(webpage)
+        mediaset_urls = MediasetIE._extract_urls(self, webpage)
         if mediaset_urls:
             return self.playlist_from_matches(
                 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 57f97409d..df3748798 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -4,6 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .theplatform import ThePlatformBaseIE
+from ..compat import (
+    compat_parse_qs,
+    compat_str,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -76,12 +81,33 @@ class MediasetIE(ThePlatformBaseIE):
     }]
 
     @staticmethod
-    def _extract_urls(webpage):
-        return [
-            mobj.group('url')
-            for mobj in re.finditer(
-                r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>https?://(?:www\.)?video\.mediaset\.it/player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=\d+.*?)\1',
-                webpage)]
+    def _extract_urls(ie, webpage):
+        def _qs(url):
+            return compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+
+        def _program_guid(qs):
+            return qs.get('programGuid', [None])[0]
+
+        entries = []
+        for mobj in re.finditer(
+                r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?video\.mediaset\.it/player/playerIFrame(?:Twitter)?\.shtml.*?)\1',
+                webpage):
+            embed_url = mobj.group('url')
+            embed_qs = _qs(embed_url)
+            program_guid = _program_guid(embed_qs)
+            if program_guid:
+                entries.append(embed_url)
+                continue
+            video_id = embed_qs.get('id', [None])[0]
+            if not video_id:
+                continue
+            urlh = ie._request_webpage(
+                embed_url, video_id, note='Following embed URL redirect')
+            embed_url = compat_str(urlh.geturl())
+            program_guid = _program_guid(_qs(embed_url))
+            if program_guid:
+                entries.append(embed_url)
+        return entries
 
     def _real_extract(self, url):
         guid = self._match_id(url)

From c17e100b96e3a1c1d2115226c9a2f166c8338031 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Sep 2018 09:27:40 +0700
Subject: [PATCH 093/510] [pluralsight] Fix subtitles extraction (closes
 #17671)

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 1257841e4..ec67381bb 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -213,7 +213,7 @@ query viewClip {
     def _get_subtitles(self, author, clip_idx, lang, name, duration, video_id):
         captions_post = {
             'a': author,
-            'cn': clip_idx,
+            'cn': int(clip_idx),
             'lc': lang,
             'm': name,
         }

From 3d3499742c88ce89de5b49dff4e96483b089fcaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Sep 2018 11:56:15 +0700
Subject: [PATCH 094/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 800ece790..15b875297 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+version <unreleased>
+
+Extractors
+* [pluralsight] Fix subtitles extraction (#17671)
+* [mediaset] Improve embed support (#17668)
++ [youtube] Add support for invidio.us (#17613)
++ [zattoo] Add support for more zattoo platform sites
+* [zattoo] Fix extraction (#17175, #17542)
+
+
 version 2018.09.18
 
 Core

From 4c89a675dd50429a1a5f5903e624db0204d2435a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Sep 2018 11:58:25 +0700
Subject: [PATCH 095/510] release 2018.09.26

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 12 ++++++++++++
 youtube_dl/version.py     |  2 +-
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a4602287a..ed3e0a157 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.26*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.09.18
+[debug] youtube-dl version 2018.09.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 15b875297..241712037 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.09.26
 
 Extractors
 * [pluralsight] Fix subtitles extraction (#17671)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9b8601751..736ab6da7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -98,6 +98,7 @@
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:iplayer:playlist**
  - **bbc.co.uk:playlist**
+ - **BBVTV**
  - **Beatport**
  - **Beeg**
  - **BehindKink**
@@ -251,6 +252,7 @@
  - **egghead:course**: egghead.io course
  - **egghead:lesson**: egghead.io lesson
  - **eHow**
+ - **EinsUndEinsTV**
  - **Einthusan**
  - **eitb.tv**
  - **EllenTube**
@@ -268,6 +270,7 @@
  - **EsriVideo**
  - **Europa**
  - **EveryonesMixtape**
+ - **EWETV**
  - **ExpoTV**
  - **Expressen**
  - **ExtremeTube**
@@ -327,6 +330,7 @@
  - **Gfycat**
  - **GiantBomb**
  - **Giga**
+ - **GlattvisionTV**
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
  - **GloboArticle**
@@ -494,6 +498,7 @@
  - **Mixer:vod**
  - **MLB**
  - **Mnet**
+ - **MNetTV**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
  - **Mojvideo**
@@ -525,6 +530,7 @@
  - **Myvi**
  - **MyVidster**
  - **MyviEmbed**
+ - **MyVisionTV**
  - **n-tv.de**
  - **natgeo**
  - **natgeo:episodeguide**
@@ -550,6 +556,7 @@
  - **netease:program**: 网易云音乐 - 电台节目
  - **netease:singer**: 网易云音乐 - 歌手
  - **netease:song**: 网易云音乐
+ - **NetPlus**
  - **Netzkino**
  - **Newgrounds**
  - **NewgroundsPlaylist**
@@ -626,6 +633,7 @@
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
+ - **OsnatelTV**
  - **PacktPub**
  - **PacktPubCourse**
  - **PandaTV**: 熊猫TV
@@ -686,6 +694,7 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
+ - **QuantumTV**
  - **Quickline**
  - **QuicklineLive**
  - **R7**
@@ -753,6 +762,7 @@
  - **safari**: safaribooksonline.com online video
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
+ - **SAKTV**
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
@@ -1035,12 +1045,14 @@
  - **vrv**
  - **vrv:series**
  - **VShare**
+ - **VTXTV**
  - **vube**: Vube.com
  - **VuClip**
  - **VVVVID**
  - **VyboryMos**
  - **Vzaar**
  - **Walla**
+ - **WalyTV**
  - **washingtonpost**
  - **washingtonpost:article**
  - **wat.tv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2b3b584a4..6f2cc31df 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.09.18'
+__version__ = '2018.09.26'

From 85cd69adcb41454f30d4db85eeb4d5cc26b6bb5d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 26 Sep 2018 08:13:16 +0100
Subject: [PATCH 096/510] [hotstar] fix extraction(closes #14694)(closes
 #14931)(closes #17637)

---
 youtube_dl/extractor/hotstar.py | 159 +++++++++++++++-----------------
 1 file changed, 73 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index d28af36ec..354ac00dc 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -1,10 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
+import hashlib
+import hmac
+import time
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -13,37 +15,40 @@ from ..utils import (
 
 
 class HotStarBaseIE(InfoExtractor):
-    _GEO_COUNTRIES = ['IN']
+    _AKAMAI_ENCRYPTION_KEY = b'\x05\xfc\x1a\x01\xca\xc9\x4b\xc4\x12\xfc\x53\x12\x07\x75\xf9\xee'
 
-    def _download_json(self, *args, **kwargs):
-        response = super(HotStarBaseIE, self)._download_json(*args, **kwargs)
-        if response['resultCode'] != 'OK':
-            if kwargs.get('fatal'):
-                raise ExtractorError(
-                    response['errorDescription'], expected=True)
-            return None
-        return response['resultObj']
-
-    def _download_content_info(self, content_id):
-        return self._download_json(
-            'https://account.hotstar.com/AVS/besc', content_id, query={
-                'action': 'GetAggregatedContentDetails',
-                'appVersion': '5.0.40',
-                'channel': 'PCTV',
-                'contentId': content_id,
-            })['contentInfo'][0]
+    def _call_api(self, path, video_id, query_name='contentId'):
+        st = int(time.time())
+        exp = st + 6000
+        auth = 'st=%d~exp=%d~acl=/*' % (st, exp)
+        auth += '~hmac=' + hmac.new(self._AKAMAI_ENCRYPTION_KEY, auth.encode(), hashlib.sha256).hexdigest()
+        response = self._download_json(
+            'https://api.hotstar.com/' + path,
+            video_id, headers={
+                'hotstarauth': auth,
+                'x-country-code': 'IN',
+                'x-platform-code': 'JIO',
+            }, query={
+                query_name: video_id,
+                'tas': 10000,
+            })
+        if response['statusCode'] != 'OK':
+            raise ExtractorError(
+                response['body']['message'], expected=True)
+        return response['body']['results']
 
 
 class HotStarIE(HotStarBaseIE):
+    IE_NAME = 'hotstar'
     _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
     _TESTS = [{
-        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
+        'url': 'https://www.hotstar.com/can-you-not-spread-rumours/1000076273',
         'info_dict': {
             'id': '1000076273',
             'ext': 'mp4',
-            'title': 'On Air With AIB',
+            'title': 'Can You Not Spread Rumours?',
             'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
-            'timestamp': 1447227000,
+            'timestamp': 1447248600,
             'upload_date': '20151111',
             'duration': 381,
         },
@@ -58,47 +63,43 @@ class HotStarIE(HotStarBaseIE):
         'url': 'http://www.hotstar.com/1000000515',
         'only_matching': True,
     }]
+    _GEO_BYPASS = False
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_data = self._download_content_info(video_id)
+        webpage = self._download_webpage(url, video_id)
+        app_state = self._parse_json(self._search_regex(
+            r'<script>window\.APP_STATE\s*=\s*({.+?})</script>',
+            webpage, 'app state'), video_id)
+        video_data = list(app_state.values())[0]['initialState']['contentData']['content']
 
-        title = video_data['episodeTitle']
+        title = video_data['title']
 
-        if video_data.get('encrypted') == 'Y':
+        if video_data.get('drmProtected'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = []
-        for f in ('JIO',):
-            format_data = self._download_json(
-                'http://getcdn.hotstar.com/AVS/besc',
-                video_id, 'Downloading %s JSON metadata' % f,
-                fatal=False, query={
-                    'action': 'GetCDN',
-                    'asJson': 'Y',
-                    'channel': f,
-                    'id': video_id,
-                    'type': 'VOD',
-                })
-            if format_data:
-                format_url = format_data.get('src')
-                if not format_url:
-                    continue
-                ext = determine_ext(format_url)
-                if ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        format_url, video_id, 'mp4',
-                        m3u8_id='hls', fatal=False))
-                elif ext == 'f4m':
-                    # produce broken files
-                    continue
-                else:
-                    formats.append({
-                        'url': format_url,
-                        'width': int_or_none(format_data.get('width')),
-                        'height': int_or_none(format_data.get('height')),
-                    })
+        format_data = self._call_api('h/v1/play', video_id)['item']
+        format_url = format_data['playbackUrl']
+        ext = determine_ext(format_url)
+        if ext == 'm3u8':
+            try:
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id='hls'))
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                    self.raise_geo_restricted(countries=['IN'])
+                raise
+        elif ext == 'f4m':
+            # produce broken files
+            pass
+        else:
+            formats.append({
+                'url': format_url,
+                'width': int_or_none(format_data.get('width')),
+                'height': int_or_none(format_data.get('height')),
+            })
         self._sort_formats(formats)
 
         return {
@@ -106,57 +107,43 @@ class HotStarIE(HotStarBaseIE):
             'title': title,
             'description': video_data.get('description'),
             'duration': int_or_none(video_data.get('duration')),
-            'timestamp': int_or_none(video_data.get('broadcastDate')),
+            'timestamp': int_or_none(video_data.get('broadcastDate') or video_data.get('startDate')),
             'formats': formats,
+            'channel': video_data.get('channelName'),
+            'channel_id': video_data.get('channelId'),
+            'series': video_data.get('showName'),
+            'season': video_data.get('seasonName'),
+            'season_number': int_or_none(video_data.get('seasonNo')),
+            'season_id': video_data.get('seasonId'),
             'episode': title,
-            'episode_number': int_or_none(video_data.get('episodeNumber')),
-            'series': video_data.get('contentTitle'),
+            'episode_number': int_or_none(video_data.get('episodeNo')),
         }
 
 
 class HotStarPlaylistIE(HotStarBaseIE):
     IE_NAME = 'hotstar:playlist'
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com/tv/[^/]+/(?P<content_id>\d+))/(?P<type>[^/]+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/[^/]+/s-\w+/list/[^/]+/t-(?P<id>\w+)'
     _TESTS = [{
-        'url': 'http://www.hotstar.com/tv/pratidaan/14982/episodes/14812/9993',
+        'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/popular-clips/t-3_2_26',
         'info_dict': {
-            'id': '14812',
+            'id': '3_2_26',
         },
-        'playlist_mincount': 75,
+        'playlist_mincount': 20,
     }, {
-        'url': 'http://www.hotstar.com/tv/pratidaan/14982/popular-clips/9998/9998',
+        'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/extras/t-2480',
         'only_matching': True,
     }]
-    _ITEM_TYPES = {
-        'episodes': 'EPISODE',
-        'popular-clips': 'CLIPS',
-    }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        base_url = mobj.group('url')
-        content_id = mobj.group('content_id')
-        playlist_type = mobj.group('type')
+        playlist_id = self._match_id(url)
 
-        content_info = self._download_content_info(content_id)
-        playlist_id = compat_str(content_info['categoryId'])
-
-        collection = self._download_json(
-            'https://search.hotstar.com/AVS/besc', playlist_id, query={
-                'action': 'SearchContents',
-                'appVersion': '5.0.40',
-                'channel': 'PCTV',
-                'moreFilters': 'series:%s;' % playlist_id,
-                'query': '*',
-                'searchOrder': 'last_broadcast_date desc,year desc,title asc',
-                'type': self._ITEM_TYPES.get(playlist_type, 'EPISODE'),
-            })
+        collection = self._call_api('o/v1/tray/find', playlist_id, 'uqId')
 
         entries = [
             self.url_result(
-                '%s/_/%s' % (base_url, video['contentId']),
+                'https://www.hotstar.com/%s' % video['contentId'],
                 ie=HotStarIE.ie_key(), video_id=video['contentId'])
-            for video in collection['response']['docs']
+            for video in collection['assets']['items']
             if video.get('contentId')]
 
         return self.playlist_result(entries, playlist_id)

From 245cbb33bcb7d519c897277e65acdcbc45335233 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Sep 2018 15:13:25 +0100
Subject: [PATCH 097/510] [spike] fix Paramount Network extraction(closes
 #17677)

---
 youtube_dl/extractor/spike.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index e76522b45..6090e0066 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -44,3 +44,10 @@ class ParamountNetworkIE(MTVServicesInfoExtractor):
 
     _FEED_URL = 'http://www.paramountnetwork.com/feeds/mrss/'
     _GEO_COUNTRIES = ['US']
+
+    def _extract_mgid(self, webpage):
+        cs = self._parse_json(self._search_regex(
+            r'window\.__DATA__\s*=\s*({.+})',
+            webpage, 'data'), None)['children']
+        c = next(c for c in cs if c.get('type') == 'VideoPlayer')
+        return c['props']['media']['video']['config']['uri']

From 85fa80d5f9d867bab706b92d15a6c9ad5b862b47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Oct 2018 21:13:43 +0700
Subject: [PATCH 098/510] [vimeo] Add another config regex (closes #17690)

---
 youtube_dl/extractor/vimeo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0a9239b62..88f4d9979 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -551,6 +551,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 else:
                     config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
                 config_re.append(r'\bvar\s+r\s*=\s*({.+?})\s*;')
+                config_re.append(r'\bconfig\s*=\s*({.+?})\s*;')
                 config = self._search_regex(config_re, webpage, 'info section',
                                             flags=re.DOTALL)
                 config = json.loads(config)

From 365343131d752bece96d2279a3e0bcd7e9f0000f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Oct 2018 21:45:24 +0700
Subject: [PATCH 099/510] [pluralsight] Fix subtitles extraction (closes
 #17726, closes #17728)

---
 youtube_dl/extractor/pluralsight.py | 34 ++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index ec67381bb..daf172570 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -210,18 +210,26 @@ query viewClip {
 
             raise ExtractorError('Unable to log in')
 
-    def _get_subtitles(self, author, clip_idx, lang, name, duration, video_id):
-        captions_post = {
-            'a': author,
-            'cn': int(clip_idx),
-            'lc': lang,
-            'm': name,
-        }
-        captions = self._download_json(
-            '%s/player/retrieve-captions' % self._API_BASE, video_id,
-            'Downloading captions JSON', 'Unable to download captions JSON',
-            fatal=False, data=json.dumps(captions_post).encode('utf-8'),
-            headers={'Content-Type': 'application/json;charset=utf-8'})
+    def _get_subtitles(self, author, clip_idx, clip_id, lang, name, duration, video_id):
+        captions = None
+        if clip_id:
+            captions = self._download_json(
+                '%s/transcript/api/v1/caption/json/%s/%s'
+                % (self._API_BASE, clip_id, lang), video_id,
+                'Downloading captions JSON', 'Unable to download captions JSON',
+                fatal=False)
+        if not captions:
+            captions_post = {
+                'a': author,
+                'cn': int(clip_idx),
+                'lc': lang,
+                'm': name,
+            }
+            captions = self._download_json(
+                '%s/player/retrieve-captions' % self._API_BASE, video_id,
+                'Downloading captions JSON', 'Unable to download captions JSON',
+                fatal=False, data=json.dumps(captions_post).encode('utf-8'),
+                headers={'Content-Type': 'application/json;charset=utf-8'})
         if captions:
             return {
                 lang: [{
@@ -413,7 +421,7 @@ query viewClip {
 
         # TODO: other languages?
         subtitles = self.extract_subtitles(
-            author, clip_idx, 'en', name, duration, display_id)
+            author, clip_idx, clip.get('clipId'), 'en', name, duration, display_id)
 
         return {
             'id': clip_id,

From 9795d93316cb63e0e2a73ee840600be7f13e19f5 Mon Sep 17 00:00:00 2001
From: Enes <enessolak99@gmail.com>
Date: Mon, 1 Oct 2018 17:48:59 +0300
Subject: [PATCH 100/510] [openload] Add support for oload.cloud (closes
 #17710)

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index d264fe206..dc01b6346 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -243,7 +243,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -307,6 +307,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.download/f/kUEfGclsU9o',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.cloud/f/4ZDnBXRWiB8',
+        'only_matching': True,
     }, {
         # Its title has not got its extension but url has it
         'url': 'https://oload.download/f/N4Otkw39VCw/Tomb.Raider.2018.HDRip.XviD.AC3-EVO.avi.mp4',

From 3c7da54c92b7560c3ceeb8b58a67d4759ed843d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Oct 2018 22:05:18 +0700
Subject: [PATCH 101/510] [jamendo] Add support for licensing.jamendo.com
 (closes #17724)

---
 youtube_dl/extractor/jamendo.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/jamendo.py b/youtube_dl/extractor/jamendo.py
index 595d7a5b7..c21827618 100644
--- a/youtube_dl/extractor/jamendo.py
+++ b/youtube_dl/extractor/jamendo.py
@@ -26,8 +26,15 @@ class JamendoBaseIE(InfoExtractor):
 
 
 class JamendoIE(JamendoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?jamendo\.com/track/(?P<id>[0-9]+)/(?P<display_id>[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            licensing\.jamendo\.com/[^/]+|
+                            (?:www\.)?jamendo\.com
+                        )
+                        /track/(?P<id>[0-9]+)/(?P<display_id>[^/?#&]+)
+                    '''
+    _TESTS = [{
         'url': 'https://www.jamendo.com/track/196219/stories-from-emona-i',
         'md5': '6e9e82ed6db98678f171c25a8ed09ffd',
         'info_dict': {
@@ -40,14 +47,19 @@ class JamendoIE(JamendoBaseIE):
             'duration': 210,
             'thumbnail': r're:^https?://.*\.jpg'
         }
-    }
+    }, {
+        'url': 'https://licensing.jamendo.com/en/track/1496667/energetic-rock',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = self._VALID_URL_RE.match(url)
         track_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, display_id)
+        webpage = self._download_webpage(
+            'https://www.jamendo.com/track/%s/%s' % (track_id, display_id),
+            display_id)
 
         title, artist, track = self._extract_meta(webpage)
 

From 66d106f2705d565675b6a94fb0e7e0bdfe132065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Oct 2018 23:29:24 +0700
Subject: [PATCH 102/510] [philharmoniedeparis] Fix extraction and add support
 for pad.philharmoniedeparis.fr (closes #17705)

---
 youtube_dl/extractor/philharmoniedeparis.py | 118 ++++++++++++--------
 1 file changed, 70 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
index f1008ae51..f723a2b3b 100644
--- a/youtube_dl/extractor/philharmoniedeparis.py
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -2,31 +2,38 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-    xpath_text,
+    try_get,
+    urljoin,
 )
 
 
 class PhilharmonieDeParisIE(InfoExtractor):
     IE_DESC = 'Philharmonie de Paris'
-    _VALID_URL = r'https?://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)|
+                            pad\.philharmoniedeparis\.fr/doc/CIMU/
+                        )
+                        (?P<id>\d+)
+                    '''
     _TESTS = [{
+        'url': 'http://pad.philharmoniedeparis.fr/doc/CIMU/1086697/jazz-a-la-villette-knower',
+        'md5': 'a0a4b195f544645073631cbec166a2c2',
+        'info_dict': {
+            'id': '1086697',
+            'ext': 'mp4',
+            'title': 'Jazz à la Villette : Knower',
+        },
+    }, {
         'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
         'info_dict': {
             'id': '1032066',
-            'ext': 'flv',
-            'title': 'md5:d1f5585d87d041d07ce9434804bc8425',
-            'timestamp': 1428179400,
-            'upload_date': '20150404',
-            'duration': 6592.278,
+            'title': 'md5:0a031b81807b3593cffa3c9a87a167a0',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
+        'playlist_mincount': 2,
     }, {
         'url': 'http://live.philharmoniedeparis.fr/Concert/1030324.html',
         'only_matching': True,
@@ -34,45 +41,60 @@ class PhilharmonieDeParisIE(InfoExtractor):
         'url': 'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=1030324&track=&lang=fr',
         'only_matching': True,
     }]
+    _LIVE_URL = 'https://live.philharmoniedeparis.fr'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        concert = self._download_xml(
-            'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=%s' % video_id,
-            video_id).find('./concert')
+        config = self._download_json(
+            '%s/otoPlayer/config.ashx' % self._LIVE_URL, video_id, query={
+                'id': video_id,
+                'lang': 'fr-FR',
+            })
 
-        formats = []
-        info_dict = {
-            'id': video_id,
-            'title': xpath_text(concert, './titre', 'title', fatal=True),
-            'formats': formats,
-        }
-
-        fichiers = concert.find('./fichiers')
-        stream = fichiers.attrib['serveurstream']
-        for fichier in fichiers.findall('./fichier'):
-            info_dict['duration'] = float_or_none(fichier.get('timecodefin'))
-            for quality, (format_id, suffix) in enumerate([('lq', ''), ('hq', '_hd')]):
-                format_url = fichier.get('url%s' % suffix)
-                if not format_url:
+        def extract_entry(source):
+            if not isinstance(source, dict):
+                return
+            title = source.get('title')
+            if not title:
+                return
+            files = source.get('files')
+            if not isinstance(files, dict):
+                return
+            format_urls = set()
+            formats = []
+            for format_id in ('mobile', 'desktop'):
+                format_url = try_get(
+                    files, lambda x: x[format_id]['file'], compat_str)
+                if not format_url or format_url in format_urls:
                     continue
-                formats.append({
-                    'url': stream,
-                    'play_path': format_url,
-                    'ext': 'flv',
-                    'format_id': format_id,
-                    'width': int_or_none(concert.get('largeur%s' % suffix)),
-                    'height': int_or_none(concert.get('hauteur%s' % suffix)),
-                    'quality': quality,
-                })
-        self._sort_formats(formats)
+                format_urls.add(format_url)
+                m3u8_url = urljoin(self._LIVE_URL, format_url)
+                formats.extend(self._extract_m3u8_formats(
+                    m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            if not formats:
+                return
+            self._sort_formats(formats)
+            return {
+                'title': title,
+                'formats': formats,
+            }
 
-        date, hour = concert.get('date'), concert.get('heure')
-        if date and hour:
-            info_dict['timestamp'] = parse_iso8601(
-                '%s-%s-%sT%s:00' % (date[0:4], date[4:6], date[6:8], hour))
-        elif date:
-            info_dict['upload_date'] = date
+        thumbnail = urljoin(self._LIVE_URL, config.get('image'))
 
-        return info_dict
+        info = extract_entry(config)
+        if info:
+            info.update({
+                'id': video_id,
+                'thumbnail': thumbnail,
+            })
+            return info
+
+        entries = []
+        for num, chapter in enumerate(config['chapters'], start=1):
+            entry = extract_entry(chapter)
+            entry['id'] = '%s-%d' % (video_id, num)
+            entries.append(entry)
+
+        return self.playlist_result(entries, video_id, config.get('title'))

From 05e7c184da85f83b254bc3d138f89b11da802bdb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 Oct 2018 06:07:06 +0100
Subject: [PATCH 103/510] [hotstar] fix extraction in python 2(closes #17696)

---
 youtube_dl/extractor/hotstar.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 354ac00dc..bf5717f1b 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -11,6 +11,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
+    try_get,
 )
 
 
@@ -72,7 +73,11 @@ class HotStarIE(HotStarBaseIE):
         app_state = self._parse_json(self._search_regex(
             r'<script>window\.APP_STATE\s*=\s*({.+?})</script>',
             webpage, 'app state'), video_id)
-        video_data = list(app_state.values())[0]['initialState']['contentData']['content']
+        video_data = {}
+        for v in app_state.values():
+            content = try_get(v, lambda x: x['initialState']['contentData']['content'], dict)
+            if content and content.get('contentId') == video_id:
+                video_data = content
 
         title = video_data['title']
 

From d98cb62e552ee74079fda2e4173a40b14faac3fe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 Oct 2018 19:43:06 +0100
Subject: [PATCH 104/510] [crunchyroll] switch to HTTPS for RpcApi(closes
 #17749)

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index af786d096..045be0ab5 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -45,7 +45,7 @@ class CrunchyrollBaseIE(InfoExtractor):
         data['req'] = 'RpcApi' + method
         data = compat_urllib_parse_urlencode(data).encode('utf-8')
         return self._download_xml(
-            'http://www.crunchyroll.com/xml/',
+            'https://www.crunchyroll.com/xml/',
             video_id, note, fatal=False, data=data, headers={
                 'Content-Type': 'application/x-www-form-urlencoded',
             })

From f60b9803a473da8e324313d01af91e5676792c77 Mon Sep 17 00:00:00 2001
From: Enes <enessolak99@gmail.com>
Date: Sat, 29 Sep 2018 13:28:56 +0300
Subject: [PATCH 105/510] [dailymotion] Fix extraction (closes #17699)

---
 youtube_dl/extractor/dailymotion.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 040f0bd02..842d9a259 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -24,6 +24,7 @@ from ..utils import (
     str_to_int,
     unescapeHTML,
     urlencode_postdata,
+    try_get,
 )
 
 
@@ -172,7 +173,12 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)
-            metadata = player['metadata']
+            metadata = try_get(
+                player, lambda x: x['metadata'], dict) or self._download_json(
+                'http://www.dailymotion.com/player/metadata/video/%s' % video_id, video_id, query={
+                    'integration': 'inline',
+                    'GK_PV5_NEON': '1',
+                })
 
             if metadata.get('error', {}).get('type') == 'password_protected':
                 password = self._downloader.params.get('videopassword')

From 0082f44a08e33712fcd33ceabab15215c962eaac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Oct 2018 02:02:58 +0700
Subject: [PATCH 106/510] [dailymotion] Improve metadata extraction (closes
 #17706)

---
 youtube_dl/extractor/dailymotion.py | 32 ++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 842d9a259..1816c559e 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -22,9 +22,11 @@ from ..utils import (
     parse_iso8601,
     sanitized_Request,
     str_to_int,
-    unescapeHTML,
-    urlencode_postdata,
     try_get,
+    unescapeHTML,
+    update_url_query,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
@@ -172,15 +174,25 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
              r'__PLAYER_CONFIG__\s*=\s*({.+?});'],
             webpage, 'player v5', default=None)
         if player_v5:
-            player = self._parse_json(player_v5, video_id)
-            metadata = try_get(
-                player, lambda x: x['metadata'], dict) or self._download_json(
-                'http://www.dailymotion.com/player/metadata/video/%s' % video_id, video_id, query={
-                    'integration': 'inline',
-                    'GK_PV5_NEON': '1',
-                })
+            player = self._parse_json(player_v5, video_id, fatal=False) or {}
+            metadata = try_get(player, lambda x: x['metadata'], dict)
+            if not metadata:
+                metadata_url = url_or_none(try_get(
+                    player, lambda x: x['context']['metadata_template_url1']))
+                if metadata_url:
+                    metadata_url = metadata_url.replace(':videoId', video_id)
+                else:
+                    metadata_url = update_url_query(
+                        'https://www.dailymotion.com/player/metadata/video/%s'
+                        % video_id, {
+                            'embedder': url,
+                            'integration': 'inline',
+                            'GK_PV5_NEON': '1',
+                        })
+                metadata = self._download_json(
+                    metadata_url, video_id, 'Downloading metadata JSON')
 
-            if metadata.get('error', {}).get('type') == 'password_protected':
+            if try_get(metadata, lambda x: x['error']['type']) == 'password_protected':
                 password = self._downloader.params.get('videopassword')
                 if password:
                     r = int(metadata['id'][1:], 36)

From 21c1a00dd7dbb9f7551ca9809a194f6380dee7a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Oct 2018 02:27:14 +0700
Subject: [PATCH 107/510] [pluralsight] Improve authentication (closes #17762)

---
 youtube_dl/extractor/pluralsight.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index daf172570..eafe56897 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -4,6 +4,7 @@ import collections
 import json
 import os
 import random
+import re
 
 from .common import InfoExtractor
 from ..compat import (
@@ -196,7 +197,10 @@ query viewClip {
         if error:
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
-        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
+        if all(not re.search(p, response) for p in (
+                r'__INITIAL_STATE__', r'["\']currentUser["\']',
+                # new layout?
+                r'>\s*Sign out\s*<')):
             BLOCKED = 'Your account has been blocked due to suspicious activity'
             if BLOCKED in response:
                 raise ExtractorError(

From 2e7ed29e3429c20e735f3f7dfceb1b13cf757037 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Oct 2018 02:29:52 +0700
Subject: [PATCH 108/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 241712037..e2757f891 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Extractors
+* [pluralsight] Improve authentication (#17762)
+* [dailymotion] Fix extraction (#17699)
+* [crunchyroll] Switch to HTTPS for RpcApi (#17749)
++ [philharmoniedeparis] Add support for pad.philharmoniedeparis.fr (#17705)
+* [philharmoniedeparis] Fix extraction (#17705)
++ [jamendo] Add support for licensing.jamendo.com (#17724)
++ [openload] Add support for oload.cloud (#17710)
+* [pluralsight] Fix subtitles extraction (#17726, #17728)
++ [vimeo] Add another config regular expression (#17690)
+* [spike] Fix Paramount Network extraction (#17677)
+* [hotstar] Fix extraction (#14694, #14931, #17637)
+
+
 version 2018.09.26
 
 Extractors

From d96f976b0c36f65894380d3d831b0520d6260c20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Oct 2018 02:31:30 +0700
Subject: [PATCH 109/510] release 2018.10.05

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ed3e0a157..058eb4321 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.09.26*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.09.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.10.05*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.10.05**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.09.26
+[debug] youtube-dl version 2018.10.05
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e2757f891..86cf489b1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.10.05
 
 Extractors
 * [pluralsight] Improve authentication (#17762)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 736ab6da7..f167a6ddc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -360,7 +360,7 @@
  - **HitRecord**
  - **HornBunny**
  - **HotNewHipHop**
- - **HotStar**
+ - **hotstar**
  - **hotstar:playlist**
  - **Howcast**
  - **HowStuffWorks**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6f2cc31df..7d3f25019 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.09.26'
+__version__ = '2018.10.05'

From c9d891f19a923f53132b49a1f5b97f344d92503c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Oct 2018 20:11:01 +0100
Subject: [PATCH 110/510] [patreon] fix extraction(closes #14502)(closes
 #10471)

---
 youtube_dl/extractor/patreon.py | 160 ++++++++++++++++++--------------
 1 file changed, 88 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 9eb027679..6f73ed68d 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -2,52 +2,63 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class PatreonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(?P<id>[^&#]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.patreon.com/creation?hid=743933',
-            'md5': 'e25505eec1053a6e6813b8ed369875cc',
-            'info_dict': {
-                'id': '743933',
-                'ext': 'mp3',
-                'title': 'Episode 166: David Smalley of Dogma Debate',
-                'uploader': 'Cognitive Dissonance Podcast',
-                'thumbnail': 're:^https?://.*$',
-            },
+    _VALID_URL = r'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.patreon.com/creation?hid=743933',
+        'md5': 'e25505eec1053a6e6813b8ed369875cc',
+        'info_dict': {
+            'id': '743933',
+            'ext': 'mp3',
+            'title': 'Episode 166: David Smalley of Dogma Debate',
+            'description': 'md5:713b08b772cd6271b9f3906683cfacdf',
+            'uploader': 'Cognitive Dissonance Podcast',
+            'thumbnail': 're:^https?://.*$',
+            'timestamp': 1406473987,
+            'upload_date': '20140727',
         },
-        {
-            'url': 'http://www.patreon.com/creation?hid=754133',
-            'md5': '3eb09345bf44bf60451b8b0b81759d0a',
-            'info_dict': {
-                'id': '754133',
-                'ext': 'mp3',
-                'title': 'CD 167 Extra',
-                'uploader': 'Cognitive Dissonance Podcast',
-                'thumbnail': 're:^https?://.*$',
-            },
+    }, {
+        'url': 'http://www.patreon.com/creation?hid=754133',
+        'md5': '3eb09345bf44bf60451b8b0b81759d0a',
+        'info_dict': {
+            'id': '754133',
+            'ext': 'mp3',
+            'title': 'CD 167 Extra',
+            'uploader': 'Cognitive Dissonance Podcast',
+            'thumbnail': 're:^https?://.*$',
         },
-        {
-            'url': 'https://www.patreon.com/creation?hid=1682498',
-            'info_dict': {
-                'id': 'SU4fj_aEMVw',
-                'ext': 'mp4',
-                'title': 'I\'m on Patreon!',
-                'uploader': 'TraciJHines',
-                'thumbnail': 're:^https?://.*$',
-                'upload_date': '20150211',
-                'description': 'md5:c5a706b1f687817a3de09db1eb93acd4',
-                'uploader_id': 'TraciJHines',
-            },
-            'params': {
-                'noplaylist': True,
-                'skip_download': True,
-            }
+        'skip': 'Patron-only content',
+    }, {
+        'url': 'https://www.patreon.com/creation?hid=1682498',
+        'info_dict': {
+            'id': 'SU4fj_aEMVw',
+            'ext': 'mp4',
+            'title': 'I\'m on Patreon!',
+            'uploader': 'TraciJHines',
+            'thumbnail': 're:^https?://.*$',
+            'upload_date': '20150211',
+            'description': 'md5:c5a706b1f687817a3de09db1eb93acd4',
+            'uploader_id': 'TraciJHines',
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': True,
         }
-    ]
+    }, {
+        'url': 'https://www.patreon.com/posts/episode-166-of-743933',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.patreon.com/posts/743933',
+        'only_matching': True,
+    }]
 
     # Currently Patreon exposes download URL via hidden CSS, so login is not
     # needed. Keeping this commented for when this inevitably changes.
@@ -78,38 +89,43 @@ class PatreonIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage).strip()
-
-        attach_fn = self._html_search_regex(
-            r'<div class="attach"><a target="_blank" href="([^"]+)">',
-            webpage, 'attachment URL', default=None)
-        embed = self._html_search_regex(
-            r'<div[^>]+id="watchCreation"[^>]*>\s*<iframe[^>]+src="([^"]+)"',
-            webpage, 'embedded URL', default=None)
-
-        if attach_fn is not None:
-            video_url = 'http://www.patreon.com' + attach_fn
-            thumbnail = self._og_search_thumbnail(webpage)
-            uploader = self._html_search_regex(
-                r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
-        elif embed is not None:
-            return self.url_result(embed)
-        else:
-            playlist = self._parse_json(self._search_regex(
-                r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
-                webpage, 'playlist JSON'),
-                video_id, transform_source=js_to_json)
-            data = playlist[0]
-            video_url = self._proto_relative_url(data['mp3'])
-            thumbnail = self._proto_relative_url(data.get('cover'))
-            uploader = data.get('artist')
-
-        return {
+        post = self._download_json(
+            'https://www.patreon.com/api/posts/' + video_id, video_id)
+        attributes = post['data']['attributes']
+        title = attributes['title'].strip()
+        image = attributes.get('image') or {}
+        info = {
             'id': video_id,
-            'url': video_url,
-            'ext': 'mp3',
             'title': title,
-            'uploader': uploader,
-            'thumbnail': thumbnail,
+            'description': clean_html(attributes.get('content')),
+            'thumbnail': image.get('large_url') or image.get('url'),
+            'timestamp': parse_iso8601(attributes.get('published_at')),
+            'like_count': int_or_none(attributes.get('like_count')),
+            'comment_count': int_or_none(attributes.get('comment_count')),
         }
+
+        for i in post.get('included', []):
+            i_type = i.get('type')
+            if i_type == 'attachment':
+                attachment_attributes = i.get('attributes') or {}
+                attachment_url = attachment_attributes.get('url')
+                if attachment_url:
+                    info.update({
+                        'url': attachment_url,
+                        'ext': determine_ext(attachment_attributes.get('name'), 'mp3'),
+                    })
+            elif i_type == 'user':
+                user_attributes = i.get('attributes')
+                if user_attributes:
+                    info.update({
+                        'uploader': user_attributes.get('full_name'),
+                        'uploader_url': user_attributes.get('url'),
+                    })
+
+        if not info.get('url'):
+            info.update({
+                '_type': 'url',
+                'url': attributes['embed']['url'],
+            })
+
+        return info

From 19a352854f5143b7cd120e990433d0fd40f617b0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Oct 2018 22:45:04 +0100
Subject: [PATCH 111/510] [patreon] extract post_file url(#17792)

---
 youtube_dl/extractor/patreon.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 6f73ed68d..426dd8121 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -104,16 +104,18 @@ class PatreonIE(InfoExtractor):
             'comment_count': int_or_none(attributes.get('comment_count')),
         }
 
+        def add_file(file_data):
+            file_url = file_data.get('url')
+            if file_url:
+                info.update({
+                    'url': file_url,
+                    'ext': determine_ext(file_data.get('name'), 'mp3'),
+                })
+
         for i in post.get('included', []):
             i_type = i.get('type')
             if i_type == 'attachment':
-                attachment_attributes = i.get('attributes') or {}
-                attachment_url = attachment_attributes.get('url')
-                if attachment_url:
-                    info.update({
-                        'url': attachment_url,
-                        'ext': determine_ext(attachment_attributes.get('name'), 'mp3'),
-                    })
+                add_file(i.get('attributes') or {})
             elif i_type == 'user':
                 user_attributes = i.get('attributes')
                 if user_attributes:
@@ -122,6 +124,9 @@ class PatreonIE(InfoExtractor):
                         'uploader_url': user_attributes.get('url'),
                     })
 
+        if not info.get('url'):
+            add_file(attributes.get('post_file') or {})
+
         if not info.get('url'):
             info.update({
                 '_type': 'url',

From 5d90a8a5f3fef73bb4ccbecd8c61583522b88d79 Mon Sep 17 00:00:00 2001
From: yonaikerlol <39972049+yonaikerlol@users.noreply.github.com>
Date: Sun, 7 Oct 2018 09:05:45 -0400
Subject: [PATCH 112/510] [openload] Add support for oload.cc

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index dc01b6346..c652603a5 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -243,7 +243,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -314,6 +314,9 @@ class OpenloadIE(InfoExtractor):
         # Its title has not got its extension but url has it
         'url': 'https://oload.download/f/N4Otkw39VCw/Tomb.Raider.2018.HDRip.XviD.AC3-EVO.avi.mp4',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.cc/embed/5NEAbI2BDSk',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From a94e7c195e261137461b546c6446033b371dfbbe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Oct 2018 11:51:40 +0100
Subject: [PATCH 113/510] [ted] fix extraction for http and rtmp formats(closes
 #5941)(closes #17572)(closes #17894)

---
 youtube_dl/extractor/ted.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 212ac80ab..f9b6aa48f 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -212,8 +212,6 @@ class TEDIE(InfoExtractor):
 
         http_url = None
         for format_id, resources in resources_.items():
-            if not isinstance(resources, dict):
-                continue
             if format_id == 'h264':
                 for resource in resources:
                     h264_url = resource.get('file')
@@ -242,6 +240,8 @@ class TEDIE(InfoExtractor):
                         'tbr': int_or_none(resource.get('bitrate')),
                     })
             elif format_id == 'hls':
+                if not isinstance(resources, dict):
+                    continue
                 stream_url = url_or_none(resources.get('stream'))
                 if not stream_url:
                     continue

From f0ee386851bb0d53801a27dafbe4e8fee5b43d88 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Oct 2018 16:26:29 +0100
Subject: [PATCH 114/510] [tv3] remove extractor(closes #10461)(closes #15339)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/tv3.py        | 34 ------------------------------
 2 files changed, 35 deletions(-)
 delete mode 100644 youtube_dl/extractor/tv3.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 464c8d690..17b576df3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1153,7 +1153,6 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv2hu import TV2HuIE
-from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tv5mondeplus import TV5MondePlusIE
 from .tva import TVAIE
diff --git a/youtube_dl/extractor/tv3.py b/youtube_dl/extractor/tv3.py
deleted file mode 100644
index 3867ec90d..000000000
--- a/youtube_dl/extractor/tv3.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class TV3IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tv3\.co\.nz/(?P<id>[^/]+)/tabid/\d+/articleID/\d+/MCat/\d+/Default\.aspx'
-    _TEST = {
-        'url': 'http://www.tv3.co.nz/MOTORSPORT-SRS-SsangYong-Hampton-Downs-Round-3/tabid/3692/articleID/121615/MCat/2915/Default.aspx',
-        'info_dict': {
-            'id': '4659127992001',
-            'ext': 'mp4',
-            'title': 'CRC Motorsport: SRS SsangYong Hampton Downs Round 3 - S2015 Ep3',
-            'description': 'SsangYong Racing Series returns for Round 3 with drivers from New Zealand and Australia taking to the grid at Hampton Downs raceway.',
-            'uploader_id': '3812193411001',
-            'upload_date': '20151213',
-            'timestamp': 1449975272,
-        },
-        'expected_warnings': [
-            'Failed to download MPD manifest'
-        ],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/3812193411001/default_default/index.html?videoId=%s'
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        brightcove_id = self._search_regex(r'<param\s*name="@videoPlayer"\s*value="(\d+)"', webpage, 'brightcove id')
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From ee5fe42e442f04e1c9a388a46e0b4552be4a56d7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Oct 2018 17:54:38 +0100
Subject: [PATCH 115/510] [brightcove:legacy] fall back to
 brightcove:new(#13912)

---
 youtube_dl/extractor/brightcove.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 14f9a14ed..5dbd71e12 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -356,7 +356,9 @@ class BrightcoveLegacyIE(InfoExtractor):
 
     def _extract_video_info(self, video_info):
         video_id = compat_str(video_info['id'])
+
         publisher_id = video_info.get('publisherId')
+
         info = {
             'id': video_id,
             'title': video_info['displayName'].strip(),
@@ -444,8 +446,16 @@ class BrightcoveLegacyIE(InfoExtractor):
                 else:
                     return ad_info
 
-        if 'url' not in info and not info.get('formats'):
-            raise ExtractorError('Unable to extract video url for %s' % video_id)
+        if not info.get('url') and not info.get('formats'):
+            uploader_id = info.get('uploader_id')
+            if uploader_id:
+                info.update({
+                    '_type': 'url',
+                    'url': 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (uploader_id, video_id),
+                    'ie_key': BrightcoveNewIE.ie_key(),
+                })
+            else:
+                raise ExtractorError('Unable to extract video url for %s' % video_id)
         return info
 
 

From 160c2773f63c72686635533bc2553634b22e7e2e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Oct 2018 18:41:57 +0100
Subject: [PATCH 116/510] [brightcove:legacy] add another fall back to
 brightcove:new

---
 youtube_dl/extractor/brightcove.py | 39 ++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5dbd71e12..40c3959fd 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -1,8 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
+import base64
 import json
+import re
+import struct
 
 from .common import InfoExtractor
 from .adobepass import AdobePassIE
@@ -310,6 +312,10 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 
+    def _brightcove_new_url_result(self, publisher_id, video_id):
+        brightcove_new_url = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (publisher_id, video_id)
+        return self.url_result(brightcove_new_url, BrightcoveNewIE.ie_key(), video_id)
+
     def _get_video_info(self, video_id, query, referer=None):
         headers = {}
         linkBase = query.get('linkBaseURL')
@@ -323,6 +329,29 @@ class BrightcoveLegacyIE(InfoExtractor):
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:
+            publisher_id = query.get('publisherId')
+            if publisher_id and publisher_id[0].isdigit():
+                publisher_id = publisher_id[0]
+            if not publisher_id:
+                valid_key = lambda key: key and ',' in key
+                player_key = query.get('playerKey')
+                if player_key and ',' in player_key[0]:
+                    player_key = player_key[0]
+                else:
+                    player_id = query.get('playerID')
+                    if player_id and player_id[0].isdigit():
+                        player_page = self._download_webpage(
+                            'http://link.brightcove.com/services/player/bcpid' + player_id[0],
+                            video_id, headers=headers, fatal=False)
+                        if player_page:
+                            player_key = self._search_regex(
+                                r'<param\s+name="playerKey"\s+value="([\w~,-]+)"',
+                                player_page, 'player key', fatal=False)
+                if player_key:
+                    enc_pub_id = player_key.split(',')[1].replace('~', '=')
+                    publisher_id = struct.unpack('>Q', base64.urlsafe_b64decode(enc_pub_id))[0]
+                if publisher_id:
+                    return self._brightcove_new_url_result(publisher_id, video_id)
             raise ExtractorError(
                 'brightcove said: %s' % error_msg, expected=True)
 
@@ -356,9 +385,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
     def _extract_video_info(self, video_info):
         video_id = compat_str(video_info['id'])
-
         publisher_id = video_info.get('publisherId')
-
         info = {
             'id': video_id,
             'title': video_info['displayName'].strip(),
@@ -449,11 +476,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         if not info.get('url') and not info.get('formats'):
             uploader_id = info.get('uploader_id')
             if uploader_id:
-                info.update({
-                    '_type': 'url',
-                    'url': 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (uploader_id, video_id),
-                    'ie_key': BrightcoveNewIE.ie_key(),
-                })
+                info.update(self._brightcove_new_url_result(uploader_id, video_id))
             else:
                 raise ExtractorError('Unable to extract video url for %s' % video_id)
         return info

From 582797d780b6f4857a5b8b6ca8c63915242c0ab9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Oct 2018 20:47:12 +0100
Subject: [PATCH 117/510] [brightcove] remove unused variable

---
 youtube_dl/extractor/brightcove.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 40c3959fd..465ae396e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -333,7 +333,6 @@ class BrightcoveLegacyIE(InfoExtractor):
             if publisher_id and publisher_id[0].isdigit():
                 publisher_id = publisher_id[0]
             if not publisher_id:
-                valid_key = lambda key: key and ',' in key
                 player_key = query.get('playerKey')
                 if player_key and ',' in player_key[0]:
                     player_key = player_key[0]

From baeabf77428ad1a6bd5a910e7be07100fcb1eadd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Oct 2018 23:19:44 +0700
Subject: [PATCH 118/510] [rutube] Use geo verification headers (closes #17897)

---
 youtube_dl/extractor/rutube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 261bcbb83..10ac8ed1f 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -103,7 +103,8 @@ class RutubeIE(RutubeBaseIE):
 
         options = self._download_json(
             'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
-            video_id, 'Downloading options JSON')
+            video_id, 'Downloading options JSON',
+            headers=self.geo_verification_headers())
 
         formats = []
         for format_id, format_url in options['video_balancer'].items():

From b99b0bcfa079a15a988cf931a3ce44bb480dfbdb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 17 Oct 2018 06:22:07 +0100
Subject: [PATCH 119/510] [cwtv] handle api errors(closes #17905)

---
 youtube_dl/extractor/cwtv.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index 224a1fb5d..f9bd535f6 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_iso8601,
@@ -66,9 +67,12 @@ class CWTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
+        data = self._download_json(
             'http://images.cwtv.com/feed/mobileapp/video-meta/apiversion_8/guid_' + video_id,
-            video_id)['video']
+            video_id)
+        if data.get('result') != 'ok':
+            raise ExtractorError(data['msg'], expected=True)
+        video_data = data['video']
         title = video_data['title']
         mpx_url = video_data.get('mpx_url') or 'http://link.theplatform.com/s/cwtv/media/guid/2703454149/%s?formats=M3U' % video_id
 

From 7d9e858132cd28b975d2174f5836d3de03f741d1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 26 Oct 2018 05:40:49 +0100
Subject: [PATCH 120/510] [viewster] reduce format requests

---
 youtube_dl/extractor/viewster.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index d5d5b4c69..6e318479c 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -130,16 +130,16 @@ class ViewsterIE(InfoExtractor):
             def concat(suffix, sep='-'):
                 return (base_format_id + '%s%s' % (sep, suffix)) if base_format_id else suffix
 
-            for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
-                media = self._download_json(
-                    'https://public-api.viewster.com/movies/%s/video' % entry_id,
-                    video_id, 'Downloading %s JSON' % concat(media_type, ' '), fatal=False, query={
-                        'mediaType': media_type,
-                        'language': audio,
-                        'subtitle': subtitle,
-                    })
-                if not media:
-                    continue
+            medias = self._download_json(
+                'https://public-api.viewster.com/movies/%s/videos' % entry_id,
+                video_id, fatal=False, query={
+                    'mediaTypes': ['application/f4m+xml', 'application/x-mpegURL', 'video/mp4'],
+                    'language': audio,
+                    'subtitle': subtitle,
+                })
+            if not medias:
+                continue
+            for media in medias:
                 video_url = media.get('Uri')
                 if not video_url:
                     continue

From 5e733b066a5fca7fe91ad5800a3e83f0a49c8fbd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 26 Oct 2018 05:41:57 +0100
Subject: [PATCH 121/510] [dailymail] fix format extraction(closes #17976)

---
 youtube_dl/extractor/dailymail.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
index af3978035..4f75a2a30 100644
--- a/youtube_dl/extractor/dailymail.py
+++ b/youtube_dl/extractor/dailymail.py
@@ -49,6 +49,9 @@ class DailyMailIE(InfoExtractor):
             'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id)
 
         video_sources = self._download_json(sources_url, video_id)
+        body = video_sources.get('body')
+        if body:
+            video_sources = body
 
         formats = []
         for rendition in video_sources['renditions']:

From 08c7d3dadec053ff5535ab2dc91f550ef4788297 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Oct 2018 22:12:54 +0700
Subject: [PATCH 122/510] [crunchyroll] Improve extraction failsafeness (closes
 #17991)

---
 youtube_dl/extractor/crunchyroll.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 045be0ab5..4a68d092b 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import xml.etree.ElementTree as etree
 import zlib
 
 from hashlib import sha1
@@ -398,7 +399,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'Downloading subtitles for ' + sub_name, data={
                     'subtitle_script_id': sub_id,
                 })
-            if sub_doc is None:
+            if not isinstance(sub_doc, etree.Element):
                 continue
             sid = sub_doc.get('id')
             iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
@@ -515,7 +516,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                         'video_quality': stream_quality,
                         'current_page': url,
                     })
-                if streamdata is not None:
+                if isinstance(streamdata, etree.Element):
                     stream_info = streamdata.find('./{default}preload/stream_info')
                     if stream_info is not None:
                         stream_infos.append(stream_info)
@@ -526,7 +527,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                         'video_format': stream_format,
                         'video_encode_quality': stream_quality,
                     })
-                if stream_info is not None:
+                if isinstance(stream_info, etree.Element):
                     stream_infos.append(stream_info)
                 for stream_info in stream_infos:
                     video_encode_id = xpath_text(stream_info, './video_encode_id')
@@ -598,10 +599,22 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         series = self._html_search_regex(
             r'(?s)<h\d[^>]+\bid=["\']showmedia_about_episode_num[^>]+>(.+?)</h\d',
             webpage, 'series', fatal=False)
-        season = xpath_text(metadata, 'series_title')
 
-        episode = xpath_text(metadata, 'episode_title') or media_metadata.get('title')
-        episode_number = int_or_none(xpath_text(metadata, 'episode_number') or media_metadata.get('episode_number'))
+        season = episode = episode_number = duration = thumbnail = None
+
+        if isinstance(metadata, etree.Element):
+            season = xpath_text(metadata, 'series_title')
+            episode = xpath_text(metadata, 'episode_title')
+            episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
+            duration = float_or_none(media_metadata.get('duration'), 1000)
+            thumbnail = xpath_text(metadata, 'episode_image_url')
+
+        if not episode:
+            episode = media_metadata.get('title')
+        if not episode_number:
+            episode_number = int_or_none(media_metadata.get('episode_number'))
+        if not thumbnail:
+            thumbnail = media_metadata.get('thumbnail', {}).get('url')
 
         season_number = int_or_none(self._search_regex(
             r'(?s)<h\d[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h\d>\s*<h4>\s*Season (\d+)',
@@ -611,8 +624,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'id': video_id,
             'title': video_title,
             'description': video_description,
-            'duration': float_or_none(media_metadata.get('duration'), 1000),
-            'thumbnail': xpath_text(metadata, 'episode_image_url') or media_metadata.get('thumbnail', {}).get('url'),
+            'duration': duration,
+            'thumbnail': thumbnail,
             'uploader': video_uploader,
             'upload_date': video_upload_date,
             'series': series,

From 022218f2f0dce112c0e2b15923c3a368bdfe4d6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Oct 2018 22:49:10 +0700
Subject: [PATCH 123/510] [ivi] Add support for ivi.tv

---
 youtube_dl/extractor/ivi.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index cb51cef2d..86c014b07 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -15,7 +15,7 @@ from ..utils import (
 class IviIE(InfoExtractor):
     IE_DESC = 'ivi.ru'
     IE_NAME = 'ivi'
-    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ivi\.(?:ru|tv)/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
     _GEO_BYPASS = False
     _GEO_COUNTRIES = ['RU']
 
@@ -65,7 +65,11 @@ class IviIE(InfoExtractor):
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
             'skip': 'Only works from Russia',
-        }
+        },
+        {
+            'url': 'https://www.ivi.tv/watch/33560/',
+            'only_matching': True,
+        },
     ]
 
     # Sorted by quality

From c901cc38e50a47c9659db534e637ac4f6a54c450 Mon Sep 17 00:00:00 2001
From: yonaikerlol <lawlietrs7@gmail.com>
Date: Sun, 28 Oct 2018 11:51:29 -0400
Subject: [PATCH 124/510] [openload] Add support for oload.icu

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index c652603a5..a91f29f5c 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -243,7 +243,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -317,6 +317,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.cc/embed/5NEAbI2BDSk',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.icu/f/-_i4y_F_Hs8',
+        'only_matching': True
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From 1fafb329849e3f07e6a6e4141bcd4547e141745c Mon Sep 17 00:00:00 2001
From: sichuan-pepper <huajiao.sichuan.pepper@gmail.com>
Date: Sun, 28 Oct 2018 01:46:32 +0900
Subject: [PATCH 125/510] [screencast] Fix extraction (closes #14590)

---
 youtube_dl/extractor/screencast.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index 62a6a8337..c6554c905 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -90,6 +90,14 @@ class ScreencastIE(InfoExtractor):
                     r'src=(.*?)(?:$|&)', video_meta,
                     'meta tag video URL', default=None)
 
+        if video_url is None:
+            video_url = self._html_search_regex(
+                r'"MediaContentUrl":"([^"]+)"', webpage, 'media content url', default=None)
+
+        if video_url is None:
+            video_url = self._html_search_meta(
+                'og:video', webpage, default=None)
+
         if video_url is None:
             raise ExtractorError('Cannot find video')
 

From a1d1c63678dcb075a8e741947c41abfee6c790a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Oct 2018 23:23:32 +0700
Subject: [PATCH 126/510] [screencast] Improve extraction (closes #14617,
 closes #17990)

---
 youtube_dl/extractor/screencast.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index c6554c905..69a0d01f3 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -92,7 +92,8 @@ class ScreencastIE(InfoExtractor):
 
         if video_url is None:
             video_url = self._html_search_regex(
-                r'"MediaContentUrl":"([^"]+)"', webpage, 'media content url', default=None)
+                r'MediaContentUrl["\']\s*:(["\'])(?P<url>(?:(?!\1).)+)\1',
+                webpage, 'video url', default=None, group='url')
 
         if video_url is None:
             video_url = self._html_search_meta(

From 4c237ab78768972e4d61d0b97fe9078d95dc4433 Mon Sep 17 00:00:00 2001
From: Alexey Trofimov <dmzkrsk@gmail.com>
Date: Fri, 26 Oct 2018 15:00:55 +0700
Subject: [PATCH 127/510] [sportbox] Fix extraction

---
 youtube_dl/extractor/sportbox.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 54497c880..9413cf27a 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -18,7 +18,7 @@ class SportBoxEmbedIE(InfoExtractor):
         'info_dict': {
             'id': '211355',
             'ext': 'mp4',
-            'title': '211355',
+            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 292,
             'view_count': int,
@@ -48,9 +48,18 @@ class SportBoxEmbedIE(InfoExtractor):
 
         wjplayer_data = self._parse_json(
             self._search_regex(
-                r'(?s)wjplayer\(({.+?})\);', webpage, 'wjplayer settings'),
+                r'(?s)var\s+playerOptions\s*=\s*({.+?});', webpage, 'wjplayer settings'),
             video_id, transform_source=js_to_json)
 
+        wjplayer_data['sources'] = self._parse_json(
+            self._search_regex(
+                r'(?s)playerOptions\.sources\s*=\s*(\[.+?\]);', webpage, 'wjplayer sources'),
+            video_id, transform_source=js_to_json)
+
+        title = self._html_search_meta(
+            ['og:title', 'twitter:title'], webpage) or self._html_search_regex(
+            r'<title>(.+?)</title>', webpage, 'title', fatal=False) or video_id
+
         formats = []
         for source in wjplayer_data['sources']:
             src = source.get('src')
@@ -71,7 +80,7 @@ class SportBoxEmbedIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_id,
+            'title': title,
             'thumbnail': wjplayer_data.get('poster'),
             'duration': int_or_none(wjplayer_data.get('duration')),
             'view_count': view_count,

From bebef109092ba2ad1b08619661aa1b51e65be1bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Oct 2018 00:19:08 +0700
Subject: [PATCH 128/510] [extractor/common] Add validation for JSON-LD URLs

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2dbf81e6e..8452125c8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -69,6 +69,7 @@ from ..utils import (
     update_url_query,
     urljoin,
     url_basename,
+    url_or_none,
     xpath_element,
     xpath_text,
     xpath_with_ns,
@@ -1213,10 +1214,10 @@ class InfoExtractor(object):
         def extract_video_object(e):
             assert e['@type'] == 'VideoObject'
             info.update({
-                'url': e.get('contentUrl'),
+                'url': url_or_none(e.get('contentUrl')),
                 'title': unescapeHTML(e.get('name')),
                 'description': unescapeHTML(e.get('description')),
-                'thumbnail': e.get('thumbnailUrl') or e.get('thumbnailURL'),
+                'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
                 'duration': parse_duration(e.get('duration')),
                 'timestamp': unified_timestamp(e.get('uploadDate')),
                 'filesize': float_or_none(e.get('contentSize')),

From 476cf548e1c6aa83686150db7abf625c6237a67f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Oct 2018 00:20:29 +0700
Subject: [PATCH 129/510] [sportbox] Improve extraction, add support for
 matchtv.ru and fix video id (closes #17978)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/generic.py    |  6 ++--
 youtube_dl/extractor/sportbox.py   | 55 ++++++++++++++++++------------
 3 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 17b576df3..f013d13c3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1043,7 +1043,7 @@ from .spike import (
 )
 from .stitcher import StitcherIE
 from .sport5 import Sport5IE
-from .sportbox import SportBoxEmbedIE
+from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
 from .springboardplatform import SpringboardPlatformIE
 from .sprout import SproutIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2a48667f0..545e03371 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -47,7 +47,7 @@ from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
 from .tvc import TVCIE
-from .sportbox import SportBoxEmbedIE
+from .sportbox import SportBoxIE
 from .smotri import SmotriIE
 from .myvi import MyviIE
 from .condenast import CondeNastIE
@@ -2636,9 +2636,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(tvc_url, 'TVC')
 
         # Look for embedded SportBox player
-        sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
+        sportbox_urls = SportBoxIE._extract_urls(webpage)
         if sportbox_urls:
-            return self.playlist_from_matches(sportbox_urls, video_id, video_title, ie='SportBoxEmbed')
+            return self.playlist_from_matches(sportbox_urls, video_id, video_title, ie=SportBoxIE.ie_key())
 
         # Look for embedded XHamster player
         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 9413cf27a..b9017fd2a 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -8,20 +8,24 @@ from ..utils import (
     determine_ext,
     int_or_none,
     js_to_json,
+    merge_dicts,
 )
 
 
-class SportBoxEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://news\.sportbox\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
+class SportBoxIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:news\.sportbox|matchtv)\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://news.sportbox.ru/vdl/player/ci/211355',
         'info_dict': {
-            'id': '211355',
+            'id': '109158',
             'ext': 'mp4',
             'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+            'description': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 292,
             'view_count': int,
+            'timestamp': 1426237001,
+            'upload_date': '20150313',
         },
         'params': {
             # m3u8 download
@@ -33,12 +37,18 @@ class SportBoxEmbedIE(InfoExtractor):
     }, {
         'url': 'https://news.sportbox.ru/vdl/player/media/193095',
         'only_matching': True,
+    }, {
+        'url': 'https://news.sportbox.ru/vdl/player/media/109158',
+        'only_matching': True,
+    }, {
+        'url': 'https://matchtv.ru/vdl/player/media/109158',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
-            r'<iframe[^>]+src="(https?://news\.sportbox\.ru/vdl/player[^"]+)"',
+            r'<iframe[^>]+src="(https?://(?:news\.sportbox|matchtv)\.ru/vdl/player[^"]+)"',
             webpage)
 
     def _real_extract(self, url):
@@ -46,22 +56,14 @@ class SportBoxEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        wjplayer_data = self._parse_json(
+        sources = self._parse_json(
             self._search_regex(
-                r'(?s)var\s+playerOptions\s*=\s*({.+?});', webpage, 'wjplayer settings'),
+                r'(?s)playerOptions\.sources(?:WithRes)?\s*=\s*(\[.+?\])\s*;\s*\n',
+                webpage, 'sources'),
             video_id, transform_source=js_to_json)
 
-        wjplayer_data['sources'] = self._parse_json(
-            self._search_regex(
-                r'(?s)playerOptions\.sources\s*=\s*(\[.+?\]);', webpage, 'wjplayer sources'),
-            video_id, transform_source=js_to_json)
-
-        title = self._html_search_meta(
-            ['og:title', 'twitter:title'], webpage) or self._html_search_regex(
-            r'<title>(.+?)</title>', webpage, 'title', fatal=False) or video_id
-
         formats = []
-        for source in wjplayer_data['sources']:
+        for source in sources:
             src = source.get('src')
             if not src:
                 continue
@@ -75,14 +77,23 @@ class SportBoxEmbedIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        player = self._parse_json(
+            self._search_regex(
+                r'(?s)playerOptions\s*=\s*({.+?})\s*;\s*\n', webpage,
+                'player options', default='{}'),
+            video_id, transform_source=js_to_json)
+        media_id = player['mediaId']
+
+        info = self._search_json_ld(webpage, media_id, default={})
+
         view_count = int_or_none(self._search_regex(
             r'Просмотров\s*:\s*(\d+)', webpage, 'view count', default=None))
 
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': wjplayer_data.get('poster'),
-            'duration': int_or_none(wjplayer_data.get('duration')),
+        return merge_dicts(info, {
+            'id': media_id,
+            'title': self._og_search_title(webpage, default=None) or media_id,
+            'thumbnail': player.get('poster'),
+            'duration': int_or_none(player.get('duration')),
             'view_count': view_count,
             'formats': formats,
-        }
+        })

From c2fe21efaaf5be47da9d88bb2a490c688bc920f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Oct 2018 00:38:06 +0700
Subject: [PATCH 130/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 86cf489b1..a21177dac 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
++ [extractor/common] Add validation for JSON-LD URLs
+
+Extractors
++ [sportbox] Add support for matchtv.ru
+* [sportbox] Fix extraction (#17978)
+* [screencast] Fix extraction (#14590, #14617, #17990)
++ [openload] Add support for oload.icu
++ [ivi] Add support for ivi.tv
+* [crunchyroll] Improve extraction failsafeness (#17991)
+* [dailymail] Fix formats extraction (#17976)
+* [viewster] Reduce format requests
+* [cwtv] Handle API errors (#17905)
++ [rutube] Use geo verification headers (#17897)
++ [brightcove:legacy] Add fallbacks to brightcove:new (#13912)
+- [tv3] Remove extractor (#10461, #15339)
+* [ted] Fix extraction for HTTP and RTMP formats (#5941, #17572, #17894)
++ [openload] Add support for oload.cc (#17823)
++ [patreon] Extract post_file URL (#17792)
+* [patreon] Fix extraction (#14502, #10471)
+
+
 version 2018.10.05
 
 Extractors

From 9ff558f67f2285a17d2a4214b5f74aeb6ce4d9b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Oct 2018 00:39:29 +0700
Subject: [PATCH 131/510] release 2018.10.29

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +--
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 058eb4321..aefed163a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.10.05*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.10.05**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.10.29**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.10.05
+[debug] youtube-dl version 2018.10.29
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a21177dac..57dbde12d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.10.29
 
 Core
 + [extractor/common] Add validation for JSON-LD URLs
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f167a6ddc..e5a6879bc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -818,7 +818,7 @@
  - **Spiegeltv**
  - **sport.francetvinfo.fr**
  - **Sport5**
- - **SportBoxEmbed**
+ - **SportBox**
  - **SportDeutschland**
  - **SpringboardPlatform**
  - **Sprout**
@@ -909,7 +909,6 @@
  - **TV2**
  - **tv2.hu**
  - **TV2Article**
- - **TV3**
  - **TV4**: tv4.se and tv4play.se
  - **TV5MondePlus**: TV5MONDE+
  - **TVA**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7d3f25019..ae9a77966 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.10.05'
+__version__ = '2018.10.29'

From 9c4a83a1bec9d7abd066a89be40e62dd36ffa67d Mon Sep 17 00:00:00 2001
From: Ali Irani <irani.2007@gmail.com>
Date: Thu, 6 Sep 2018 02:08:38 +0430
Subject: [PATCH 132/510] [aparat] Fix extraction

---
 youtube_dl/extractor/aparat.py | 36 +++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 6eb8bbb6e..780439e17 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -34,32 +34,32 @@ class AparatIE(InfoExtractor):
             'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
             video_id)
 
-        title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
-
         file_list = self._parse_json(
             self._search_regex(
-                r'fileList\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage,
+                r'var options\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage,
                 'file list'),
             video_id)
 
+        title = file_list['plugins']['sabaPlayerPlugin']['title']
+
         formats = []
-        for item in file_list[0]:
-            file_url = url_or_none(item.get('file'))
-            if not file_url:
-                continue
-            ext = mimetype2ext(item.get('type'))
-            label = item.get('label')
-            formats.append({
-                'url': file_url,
-                'ext': ext,
-                'format_id': label or ext,
-                'height': int_or_none(self._search_regex(
-                    r'(\d+)[pP]', label or '', 'height', default=None)),
-            })
+        for list in file_list['plugins']['sabaPlayerPlugin']['multiSRC']:
+            for item in list:
+                file_url = url_or_none(item.get('src'))
+                if not file_url:
+                    continue
+                ext = mimetype2ext(item.get('type'))
+                label = item.get('label')
+                formats.append({
+                    'url': file_url,
+                    'ext': ext,
+                    'format_id': label or ext,
+                    'height': int_or_none(self._search_regex(
+                        r'(\d+)[pP]', label or '', 'height', default=None)),
+                })
         self._sort_formats(formats)
 
-        thumbnail = self._search_regex(
-            r'image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
+        thumbnail = file_list['poster']
 
         return {
             'id': video_id,

From 2943397e8701d3dcd28433e485e50459fdbda62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Oct 2018 23:29:05 +0700
Subject: [PATCH 133/510] [aparat] Improve extraction and extract more metadata
 (closes #17445, closes #18008)

---
 youtube_dl/extractor/aparat.py | 89 ++++++++++++++++++++++------------
 1 file changed, 57 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 780439e17..883dcee7a 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    merge_dicts,
     mimetype2ext,
     url_or_none,
 )
@@ -12,59 +13,83 @@ from ..utils import (
 class AparatIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.aparat.com/v/wP8On',
         'md5': '131aca2e14fe7c4dcb3c4877ba300c89',
         'info_dict': {
             'id': 'wP8On',
             'ext': 'mp4',
             'title': 'تیم گلکسی 11 - زومیت',
-            'age_limit': 0,
+            'description': 'md5:096bdabcdcc4569f2b8a5e903a3b3028',
+            'duration': 231,
+            'timestamp': 1387394859,
+            'upload_date': '20131218',
+            'view_count': int,
         },
-        # 'skip': 'Extremely unreliable',
-    }
+    }, {
+        # multiple formats
+        'url': 'https://www.aparat.com/v/8dflw/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Note: There is an easier-to-parse configuration at
-        # http://www.aparat.com/video/video/config/videohash/%video_id
-        # but the URL in there does not work
-        webpage = self._download_webpage(
-            'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
-            video_id)
+        # Provides more metadata
+        webpage = self._download_webpage(url, video_id, fatal=False)
 
-        file_list = self._parse_json(
+        if not webpage:
+            # Note: There is an easier-to-parse configuration at
+            # http://www.aparat.com/video/video/config/videohash/%video_id
+            # but the URL in there does not work
+            webpage = self._download_webpage(
+                'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
+                video_id)
+
+        options = self._parse_json(
             self._search_regex(
-                r'var options\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage,
-                'file list'),
+                r'options\s*=\s*JSON\.parse\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1\s*\)',
+                webpage, 'options', group='value'),
             video_id)
 
-        title = file_list['plugins']['sabaPlayerPlugin']['title']
+        player = options['plugins']['sabaPlayerPlugin']
 
         formats = []
-        for list in file_list['plugins']['sabaPlayerPlugin']['multiSRC']:
-            for item in list:
+        for sources in player['multiSRC']:
+            for item in sources:
+                if not isinstance(item, dict):
+                    continue
                 file_url = url_or_none(item.get('src'))
                 if not file_url:
                     continue
-                ext = mimetype2ext(item.get('type'))
-                label = item.get('label')
-                formats.append({
-                    'url': file_url,
-                    'ext': ext,
-                    'format_id': label or ext,
-                    'height': int_or_none(self._search_regex(
-                        r'(\d+)[pP]', label or '', 'height', default=None)),
-                })
-        self._sort_formats(formats)
+                item_type = item.get('type')
+                if item_type == 'application/vnd.apple.mpegurl':
+                    formats.extend(self._extract_m3u8_formats(
+                        file_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls',
+                        fatal=False))
+                else:
+                    ext = mimetype2ext(item.get('type'))
+                    label = item.get('label')
+                    formats.append({
+                        'url': file_url,
+                        'ext': ext,
+                        'format_id': 'http-%s' % (label or ext),
+                        'height': int_or_none(self._search_regex(
+                            r'(\d+)[pP]', label or '', 'height',
+                            default=None)),
+                    })
+        self._sort_formats(
+            formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
-        thumbnail = file_list['poster']
+        info = self._search_json_ld(webpage, video_id, default={})
 
-        return {
+        if not info.get('title'):
+            info['title'] = player['title']
+
+        return merge_dicts(info, {
             'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'age_limit': self._family_friendly_search(webpage),
+            'thumbnail': url_or_none(options.get('poster')),
+            'duration': int_or_none(player.get('duration')),
             'formats': formats,
-        }
+        })

From ffa7b2bfee7b94191ffc20ef00c22f708c97cddf Mon Sep 17 00:00:00 2001
From: gfabiano <gfabiano40@gmail.com>
Date: Mon, 30 Jul 2018 18:15:20 +0200
Subject: [PATCH 134/510] [cbnc] Add support for new URL schema (closes #14193)

---
 youtube_dl/extractor/cnbc.py       | 41 +++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |  5 +++-
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dl/extractor/cnbc.py
index d354d9f95..35c0b6124 100644
--- a/youtube_dl/extractor/cnbc.py
+++ b/youtube_dl/extractor/cnbc.py
@@ -1,8 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+
 from .common import InfoExtractor
-from ..utils import smuggle_url
+from ..utils import (
+    js_to_json,
+    smuggle_url,
+)
 
 
 class CNBCIE(InfoExtractor):
@@ -34,3 +38,38 @@ class CNBCIE(InfoExtractor):
                 {'force_smil_url': True}),
             'id': video_id,
         }
+
+
+class CNBCNewIE(InfoExtractor):
+    IE_NAME = 'CNBC:new'
+    _VALID_URL = r'https?://(?:www)?\.cnbc\.com/video.*/(?P<id>[^.]+)'  
+    _TEST = {
+        'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html',
+        'info_dict': {
+            'id': '7000031301',
+            'ext': 'mp4',
+            'title': 'Trump: I don\'t necessarily agree with raising rates',
+            'description': 'md5:878d8f0b4ebb5bb1dda3514b91b49de3',
+            'timestamp': 1531958400,
+            'upload_date': '20180719',
+            'uploader': 'NBCU-CNBC',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    CNBC_URL_TEMPLATE = 'http://video.cnbc.com/gallery/?video=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._parse_json(
+            self._search_regex(
+                r'(?s).*<script[^>]*>.*?({.+?content_id.+?}).*?</script>',
+                webpage, display_id),
+            display_id, transform_source=js_to_json
+        )['content_id']
+
+        return self.url_result(self.CNBC_URL_TEMPLATE % video_id, 'CNBC')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f013d13c3..93574907b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -209,7 +209,10 @@ from .cloudy import CloudyIE
 from .clubic import ClubicIE
 from .clyp import ClypIE
 from .cmt import CMTIE
-from .cnbc import CNBCIE
+from .cnbc import (
+    CNBCIE,
+    CNBCNewIE,
+)
 from .cnn import (
     CNNIE,
     CNNBlogsIE,

From 94db1f7f3b7269d5843b815ef2aa5b71d0361e6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Oct 2018 23:53:39 +0700
Subject: [PATCH 135/510] [cnbc] Simplify extraction (closes #14280, closes
 #17110)

---
 youtube_dl/extractor/cnbc.py       | 29 ++++++++++-------------------
 youtube_dl/extractor/extractors.py |  2 +-
 2 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dl/extractor/cnbc.py
index 35c0b6124..81b0c9fc4 100644
--- a/youtube_dl/extractor/cnbc.py
+++ b/youtube_dl/extractor/cnbc.py
@@ -3,10 +3,7 @@ from __future__ import unicode_literals
 
 
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    smuggle_url,
-)
+from ..utils import smuggle_url
 
 
 class CNBCIE(InfoExtractor):
@@ -40,36 +37,30 @@ class CNBCIE(InfoExtractor):
         }
 
 
-class CNBCNewIE(InfoExtractor):
-    IE_NAME = 'CNBC:new'
-    _VALID_URL = r'https?://(?:www)?\.cnbc\.com/video.*/(?P<id>[^.]+)'  
+class CNBCVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www)?\.cnbc\.com/video/(?:[^/]+/)+(?P<id>[^./?#&]+)'
     _TEST = {
         'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html',
         'info_dict': {
             'id': '7000031301',
             'ext': 'mp4',
-            'title': 'Trump: I don\'t necessarily agree with raising rates',
+            'title': "Trump: I don't necessarily agree with raising rates",
             'description': 'md5:878d8f0b4ebb5bb1dda3514b91b49de3',
             'timestamp': 1531958400,
             'upload_date': '20180719',
             'uploader': 'NBCU-CNBC',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
     }
 
-    CNBC_URL_TEMPLATE = 'http://video.cnbc.com/gallery/?video=%s'
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._parse_json(
-            self._search_regex(
-                r'(?s).*<script[^>]*>.*?({.+?content_id.+?}).*?</script>',
-                webpage, display_id),
-            display_id, transform_source=js_to_json
-        )['content_id']
-
-        return self.url_result(self.CNBC_URL_TEMPLATE % video_id, 'CNBC')
+        video_id = self._search_regex(
+            r'content_id["\']\s*:\s*["\'](\d+)', webpage, display_id,
+            'video id')
+        return self.url_result(
+            'http://video.cnbc.com/gallery/?video=%s' % video_id,
+            CNBCIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 93574907b..d96e23905 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -211,7 +211,7 @@ from .clyp import ClypIE
 from .cmt import CMTIE
 from .cnbc import (
     CNBCIE,
-    CNBCNewIE,
+    CNBCVideoIE,
 )
 from .cnn import (
     CNNIE,

From 9aac22c195cf41ff46a644bd027481629d0e6d06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Oct 2018 00:22:18 +0700
Subject: [PATCH 136/510] [theplatform] Improve error detection (#13222)

---
 youtube_dl/extractor/theplatform.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index ffef5bf06..181620615 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -39,9 +39,17 @@ class ThePlatformBaseIE(OnceIE):
             smil_url, video_id, note=note, query={'format': 'SMIL'},
             headers=self.geo_verification_headers())
         error_element = find_xpath_attr(meta, _x('.//smil:ref'), 'src')
-        if error_element is not None and error_element.attrib['src'].startswith(
-                'http://link.theplatform.%s/s/errorFiles/Unavailable.' % self._TP_TLD):
-            raise ExtractorError(error_element.attrib['abstract'], expected=True)
+        if error_element is not None:
+            exception = find_xpath_attr(
+                error_element, _x('.//smil:param'), 'name', 'exception')
+            if exception is not None:
+                if exception.get('value') == 'GeoLocationBlocked':
+                    self.raise_geo_restricted(error_element.attrib['abstract'])
+                elif error_element.attrib['src'].startswith(
+                        'http://link.theplatform.%s/s/errorFiles/Unavailable.'
+                        % self._TP_TLD):
+                    raise ExtractorError(
+                        error_element.attrib['abstract'], expected=True)
 
         smil_formats = self._parse_smil_formats(
             meta, smil_url, video_id, namespace=default_ns,

From aa7e974a2a61a20e017b52a3a9ab1fb43cf8cd13 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Oct 2018 19:28:09 +0100
Subject: [PATCH 137/510] [linkedin:learning] Add new extractor(closes #13545)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/linkedin.py   | 175 +++++++++++++++++++++++++++++
 2 files changed, 179 insertions(+)
 create mode 100644 youtube_dl/extractor/linkedin.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d96e23905..8879f5d90 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -572,6 +572,10 @@ from .limelight import (
     LimelightChannelListIE,
 )
 from .line import LineTVIE
+from .linkedin import (
+    LinkedInLearningIE,
+    LinkedInLearningCourseIE,
+)
 from .litv import LiTVIE
 from .liveleak import (
     LiveLeakIE,
diff --git a/youtube_dl/extractor/linkedin.py b/youtube_dl/extractor/linkedin.py
new file mode 100644
index 000000000..6333a8fd3
--- /dev/null
+++ b/youtube_dl/extractor/linkedin.py
@@ -0,0 +1,175 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    urlencode_postdata,
+)
+
+
+class LinkedInLearningBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'linkedin'
+
+    def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
+        query = {
+            'courseSlug': course_slug,
+            'fields': fields,
+            'q': 'slugs',
+        }
+        sub = ''
+        if video_slug:
+            query.update({
+                'videoSlug': video_slug,
+                'resolution': '_%s' % resolution,
+            })
+            sub = ' %dp' % resolution
+        api_url = 'https://www.linkedin.com/learning-api/detailedCourses'
+        return self._download_json(
+            api_url, video_slug, 'Downloading%s JSON metadata' % sub, headers={
+                'Csrf-Token': self._get_cookies(api_url)['JSESSIONID'].value,
+            }, query=query)['elements'][0]
+
+    def _get_video_id(self, urn, course_slug, video_slug):
+        if urn:
+            mobj = re.search(r'urn:li:lyndaCourse:\d+,(\d+)', urn)
+            if mobj:
+                return mobj.group(1)
+        return '%s/%s' % (course_slug, video_slug)
+
+    def _real_initialize(self):
+        email, password = self._get_login_info()
+        if email is None:
+            return
+
+        login_page = self._download_webpage(
+            'https://www.linkedin.com/uas/login?trk=learning',
+            None, 'Downloading login page')
+        action_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, 'post url',
+            default='https://www.linkedin.com/uas/login-submit', group='url')
+        data = self._hidden_inputs(login_page)
+        data.update({
+            'session_key': email,
+            'session_password': password,
+        })
+        login_submit_page = self._download_webpage(
+            action_url, None, 'Logging in',
+            data=urlencode_postdata(data))
+        error = self._search_regex(
+            r'<span[^>]+class="error"[^>]*>\s*(.+?)\s*</span>',
+            login_submit_page, 'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
+
+class LinkedInLearningIE(LinkedInLearningBaseIE):
+    IE_NAME = 'linkedin:learning'
+    _VALID_URL = r'https?://(?:www\.)?linkedin\.com/learning/(?P<course_slug>[^/]+)/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://www.linkedin.com/learning/programming-foundations-fundamentals/welcome?autoplay=true',
+        'md5': 'a1d74422ff0d5e66a792deb996693167',
+        'info_dict': {
+            'id': '90426',
+            'ext': 'mp4',
+            'title': 'Welcome',
+            'timestamp': 1430396150.82,
+            'upload_date': '20150430',
+        },
+    }
+
+    def _real_extract(self, url):
+        course_slug, video_slug = re.match(self._VALID_URL, url).groups()
+
+        video_data = None
+        formats = []
+        for width, height in ((640, 360), (960, 540), (1280, 720)):
+            video_data = self._call_api(
+                course_slug, 'selectedVideo', video_slug, height)['selectedVideo']
+
+            video_url_data = video_data.get('url') or {}
+            progressive_url = video_url_data.get('progressiveUrl')
+            if progressive_url:
+                formats.append({
+                    'format_id': 'progressive-%dp' % height,
+                    'url': progressive_url,
+                    'height': height,
+                    'width': width,
+                    'source_preference': 1,
+                })
+
+        title = video_data['title']
+
+        audio_url = video_data.get('audio', {}).get('progressiveUrl')
+        if audio_url:
+            formats.append({
+                'abr': 64,
+                'ext': 'm4a',
+                'format_id': 'audio',
+                'url': audio_url,
+                'vcodec': 'none',
+            })
+
+        streaming_url = video_url_data.get('streamingUrl')
+        if streaming_url:
+            formats.extend(self._extract_m3u8_formats(
+                streaming_url, video_slug, 'mp4',
+                'm3u8_native', m3u8_id='hls', fatal=False))
+
+        self._sort_formats(formats, ('width', 'height', 'source_preference', 'tbr', 'abr'))
+
+        return {
+            'id': self._get_video_id(video_data.get('urn'), course_slug, video_slug),
+            'title': title,
+            'formats': formats,
+            'thumbnail': video_data.get('defaultThumbnail'),
+            'timestamp': float_or_none(video_data.get('publishedOn'), 1000),
+            'duration': int_or_none(video_data.get('durationInSeconds')),
+        }
+
+
+class LinkedInLearningCourseIE(LinkedInLearningBaseIE):
+    IE_NAME = 'linkedin:learning:course'
+    _VALID_URL = r'https?://(?:www\.)?linkedin\.com/learning/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://www.linkedin.com/learning/programming-foundations-fundamentals',
+        'info_dict': {
+            'id': 'programming-foundations-fundamentals',
+            'title': 'Programming Foundations: Fundamentals',
+            'description': 'md5:76e580b017694eb89dc8e8923fff5c86',
+        },
+        'playlist_mincount': 61,
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if LinkedInLearningIE.suitable(url) else super(LinkedInLearningCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        course_slug = self._match_id(url)
+        course_data = self._call_api(course_slug, 'chapters,description,title')
+
+        entries = []
+        for chapter in course_data.get('chapters', []):
+            chapter_title = chapter.get('title')
+            for video in chapter.get('videos', []):
+                video_slug = video.get('slug')
+                if not video_slug:
+                    continue
+                entries.append({
+                    '_type': 'url',
+                    'id': self._get_video_id(video.get('urn'), course_slug, video_slug),
+                    'title': video.get('title'),
+                    'url': 'https://www.linkedin.com/learning/%s/%s' % (course_slug, video_slug),
+                    'chapter': chapter_title,
+                    'ie_key': LinkedInLearningIE.ie_key(),
+                })
+
+        return self.playlist_result(
+            entries, course_slug,
+            course_data.get('title'),
+            course_data.get('description'))

From b14475724b78bf3b4f2f448bb2953dfd52d3d425 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Oct 2018 21:49:12 +0100
Subject: [PATCH 138/510] [linkedin:learning:course] use url_transparent type
 for playlist entries

---
 youtube_dl/extractor/linkedin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/linkedin.py b/youtube_dl/extractor/linkedin.py
index 6333a8fd3..259fc4c5e 100644
--- a/youtube_dl/extractor/linkedin.py
+++ b/youtube_dl/extractor/linkedin.py
@@ -161,7 +161,7 @@ class LinkedInLearningCourseIE(LinkedInLearningBaseIE):
                 if not video_slug:
                     continue
                 entries.append({
-                    '_type': 'url',
+                    '_type': 'url_transparent',
                     'id': self._get_video_id(video.get('urn'), course_slug, video_slug),
                     'title': video.get('title'),
                     'url': 'https://www.linkedin.com/learning/%s/%s' % (course_slug, video_slug),

From f16679e8436fb0e9d01aca2343ce22a01802667f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Oct 2018 04:57:28 +0700
Subject: [PATCH 139/510] [cnbc:video] Fix _VALID_URL (#17110)

---
 youtube_dl/extractor/cnbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dl/extractor/cnbc.py
index 81b0c9fc4..6889b0f40 100644
--- a/youtube_dl/extractor/cnbc.py
+++ b/youtube_dl/extractor/cnbc.py
@@ -38,7 +38,7 @@ class CNBCIE(InfoExtractor):
 
 
 class CNBCVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www)?\.cnbc\.com/video/(?:[^/]+/)+(?P<id>[^./?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?cnbc\.com/video/(?:[^/]+/)+(?P<id>[^./?#&]+)'
     _TEST = {
         'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html',
         'info_dict': {

From c70ba664f19f0323d74e4e8ea76249f4c97def06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Nov 2018 01:35:32 +0700
Subject: [PATCH 140/510] [njpwworld] Fix authentication (closes #17427)

---
 youtube_dl/extractor/njpwworld.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/njpwworld.py b/youtube_dl/extractor/njpwworld.py
index febef097a..025c5d249 100644
--- a/youtube_dl/extractor/njpwworld.py
+++ b/youtube_dl/extractor/njpwworld.py
@@ -31,6 +31,8 @@ class NJPWWorldIE(InfoExtractor):
         'skip': 'Requires login',
     }
 
+    _LOGIN_URL = 'https://front.njpwworld.com/auth/login'
+
     def _real_initialize(self):
         self._login()
 
@@ -40,13 +42,17 @@ class NJPWWorldIE(InfoExtractor):
         if not username:
             return True
 
+        # Setup session (will set necessary cookies)
+        self._request_webpage(
+            'https://njpwworld.com/', None, note='Setting up session')
+
         webpage, urlh = self._download_webpage_handle(
-            'https://njpwworld.com/auth/login', None,
+            self._LOGIN_URL, None,
             note='Logging in', errnote='Unable to login',
             data=urlencode_postdata({'login_id': username, 'pw': password}),
-            headers={'Referer': 'https://njpwworld.com/auth'})
+            headers={'Referer': 'https://front.njpwworld.com/auth'})
         # /auth/login will return 302 for successful logins
-        if urlh.geturl() == 'https://njpwworld.com/auth/login':
+        if urlh.geturl() == self._LOGIN_URL:
             self.report_warning('unable to login')
             return False
 

From 061ea3a776830f15dbce899bad8b48232f32aaf0 Mon Sep 17 00:00:00 2001
From: yonaikerlol <lawlietrs7@gmail.com>
Date: Fri, 2 Nov 2018 12:08:41 -0400
Subject: [PATCH 141/510] [openload] Add support for oload.fun

---
 youtube_dl/extractor/openload.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index a91f29f5c..2473536fd 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -243,7 +243,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -319,7 +319,10 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'https://oload.icu/f/-_i4y_F_Hs8',
-        'only_matching': True
+        'only_matching': True,
+    }, {
+        'url': 'https://oload.fun/f/gb6G1H4sHXY',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From c620694c97151396055108cd10d2a393036eb334 Mon Sep 17 00:00:00 2001
From: Sebastian Haas <sehaas@deebas.com>
Date: Tue, 30 Oct 2018 23:44:50 +0100
Subject: [PATCH 142/510] [orf:tvthek] Fix extraction (closes #17737)

use _extract_m3u8_formats and _extract_f4m_formats helper functions
closes #17737
---
 youtube_dl/extractor/orf.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index c1fb580ca..da8031ad2 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -80,14 +80,16 @@ class ORFTVthekIE(InfoExtractor):
             if not video_id or not title:
                 continue
             video_id = compat_str(video_id)
-            formats = [{
-                'preference': -10 if fd['delivery'] == 'hls' else None,
-                'format_id': '%s-%s-%s' % (
-                    fd['delivery'], fd['quality'], fd['quality_string']),
-                'url': fd['src'],
-                'protocol': fd['protocol'],
-                'quality': quality_to_int(fd['quality']),
-            } for fd in sd['sources']]
+            formats = []
+            for fd in sd['sources']:
+                format_id = '%s-%s-%s' % (
+                    fd['delivery'], fd['quality'], fd['quality_string'])
+                if determine_ext(fd['src']) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        fd['src'], video_id, 'mp4', m3u8_id=format_id))
+                elif determine_ext(fd['src']) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        fd['src'], video_id, f4m_id=format_id))
 
             # Check for geoblocking.
             # There is a property is_geoprotection, but that's always false

From 4b6aca17cc7d4df22e78501b4c00a9281c189ab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Nov 2018 23:46:56 +0700
Subject: [PATCH 143/510] [orf:tvthek] Improve extraction and remove unused
 code (closes #17956, closes #18024)

---
 youtube_dl/extractor/orf.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index da8031ad2..d432e3449 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -15,6 +15,7 @@ from ..utils import (
     strip_jsonp,
     unescapeHTML,
     unified_strdate,
+    url_or_none,
 )
 
 
@@ -68,12 +69,6 @@ class ORFTVthekIE(InfoExtractor):
                 webpage, 'playlist', group='json'),
             playlist_id, transform_source=unescapeHTML)['playlist']['videos']
 
-        def quality_to_int(s):
-            m = re.search('([0-9]+)', s)
-            if m is None:
-                return -1
-            return int(m.group(1))
-
         entries = []
         for sd in data_jsb:
             video_id, title = sd.get('id'), sd.get('title')
@@ -82,14 +77,27 @@ class ORFTVthekIE(InfoExtractor):
             video_id = compat_str(video_id)
             formats = []
             for fd in sd['sources']:
-                format_id = '%s-%s-%s' % (
-                    fd['delivery'], fd['quality'], fd['quality_string'])
+                src = url_or_none(fd.get('src'))
+                if not src:
+                    continue
+                format_id_list = []
+                for key in ('delivery', 'quality', 'quality_string'):
+                    value = fd.get(key)
+                    if value:
+                        format_id_list.append(value)
+                format_id = '-'.join(format_id_list)
                 if determine_ext(fd['src']) == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         fd['src'], video_id, 'mp4', m3u8_id=format_id))
                 elif determine_ext(fd['src']) == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         fd['src'], video_id, f4m_id=format_id))
+                else:
+                    formats.append({
+                        'format_id': format_id,
+                        'url': src,
+                        'protocol': fd.get('protocol'),
+                    })
 
             # Check for geoblocking.
             # There is a property is_geoprotection, but that's always false

From 036f905161b104ef90e75ad42d472b45eeb102ba Mon Sep 17 00:00:00 2001
From: sichuan-pepper <huajiao.sichuan.pepper@gmail.com>
Date: Sat, 27 Oct 2018 03:40:44 +0900
Subject: [PATCH 144/510] [twitcasting] Add extractor

---
 youtube_dl/extractor/extractors.py  |  1 +
 youtube_dl/extractor/twitcasting.py | 44 +++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 youtube_dl/extractor/twitcasting.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8879f5d90..27452d73e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1196,6 +1196,7 @@ from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
 from .twentythreevideo import TwentyThreeVideoIE
+from .twitcasting import TwitcastingIE
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,
diff --git a/youtube_dl/extractor/twitcasting.py b/youtube_dl/extractor/twitcasting.py
new file mode 100644
index 000000000..856df5c0b
--- /dev/null
+++ b/youtube_dl/extractor/twitcasting.py
@@ -0,0 +1,44 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import re
+
+
+class TwitcastingIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|ssl|en|pt|es|ja|ko)\.)?twitcasting\.tv/(?P<uploader_id>[^\/]+)/movie/(?P<video_id>[0-9]+)'
+    _TEST = {
+        'url': 'https://twitcasting.tv/ivetesangalo/movie/2357609',
+        'md5': '745243cad58c4681dc752490f7540d7f',
+        'info_dict': {
+            'id': '2357609',
+            'ext': 'mp4',
+            'title': 'Recorded Live #2357609',
+            'uploader_id': 'ivetesangalo',
+            'description': "Moi! I'm live on TwitCasting from my iPhone.",
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+        uploader_id = mobj.group('uploader_id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        playlist_url = self._html_search_regex(r'(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage, name='playlist url', group='url')
+        formats = self._extract_m3u8_formats(playlist_url, video_id, ext='mp4')
+        thumbnail = self._og_search_thumbnail(webpage)
+        title = self._html_search_meta('twitter:title', webpage)
+        description = self._og_search_description(webpage) or self._html_search_meta('twitter:description', webpage)
+        return{
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader_id': uploader_id,
+            'formats': formats,
+        }

From cf0db4d99785532d767d0ca1cc029c73d16bb045 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Nov 2018 00:27:36 +0700
Subject: [PATCH 145/510] [twitcasting] Improve extraction and fix issues
 (closes #17981)

---
 youtube_dl/extractor/extractors.py  |  2 +-
 youtube_dl/extractor/twitcasting.py | 36 +++++++++++++++++++++--------
 2 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 27452d73e..b41cd65d7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1196,7 +1196,7 @@ from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
 from .twentythreevideo import TwentyThreeVideoIE
-from .twitcasting import TwitcastingIE
+from .twitcasting import TwitCastingIE
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,
diff --git a/youtube_dl/extractor/twitcasting.py b/youtube_dl/extractor/twitcasting.py
index 856df5c0b..05f8aa9ce 100644
--- a/youtube_dl/extractor/twitcasting.py
+++ b/youtube_dl/extractor/twitcasting.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 import re
 
 
-class TwitcastingIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|ssl|en|pt|es|ja|ko)\.)?twitcasting\.tv/(?P<uploader_id>[^\/]+)/movie/(?P<video_id>[0-9]+)'
+class TwitCastingIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^/]+\.)?twitcasting\.tv/(?P<uploader_id>[^/]+)/movie/(?P<id>\d+)'
     _TEST = {
         'url': 'https://twitcasting.tv/ivetesangalo/movie/2357609',
         'md5': '745243cad58c4681dc752490f7540d7f',
@@ -18,24 +18,40 @@ class TwitcastingIE(InfoExtractor):
             'uploader_id': 'ivetesangalo',
             'description': "Moi! I'm live on TwitCasting from my iPhone.",
             'thumbnail': r're:^https?://.*\.jpg$',
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = mobj.group('id')
         uploader_id = mobj.group('uploader_id')
 
         webpage = self._download_webpage(url, video_id)
 
-        playlist_url = self._html_search_regex(r'(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage, name='playlist url', group='url')
-        formats = self._extract_m3u8_formats(playlist_url, video_id, ext='mp4')
+        title = self._html_search_regex(
+            r'(?s)<[^>]+id=["\']movietitle[^>]+>(.+?)</',
+            webpage, 'title', default=None) or self._html_search_meta(
+            'twitter:title', webpage, fatal=True)
+
+        m3u8_url = self._search_regex(
+            (r'data-movie-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+             r'(["\'])(?P<url>http.+?\.m3u8.*?)\1'),
+            webpage, 'm3u8 url', group='url')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
         thumbnail = self._og_search_thumbnail(webpage)
-        title = self._html_search_meta('twitter:title', webpage)
-        description = self._og_search_description(webpage) or self._html_search_meta('twitter:description', webpage)
-        return{
+        description = self._og_search_description(
+            webpage, default=None) or self._html_search_meta(
+            'twitter:description', webpage)
+
+        return {
             'id': video_id,
-            'url': url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 95e42d7336d01f505d6551a21df52f3ae234e96b Mon Sep 17 00:00:00 2001
From: Xiao Di Guan <xdg@puxlit.net>
Date: Sat, 3 Nov 2018 05:18:20 +1100
Subject: [PATCH 146/510] [extractor/common] Ensure response handle is not
 prematurely closed before it can be read if it matches expected_status
 (resolves #17195, closes #17846, resolves #17447)

---
 test/helper.py                 | 10 ++++++++
 test/test_InfoExtractor.py     | 42 ++++++++++++++++++++++++++++++++--
 test/test_downloader_http.py   | 12 +---------
 test/test_http.py              | 10 +-------
 youtube_dl/extractor/common.py |  5 ++++
 5 files changed, 57 insertions(+), 22 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index dfee217a9..aa9a1c9b2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -7,6 +7,7 @@ import json
 import os.path
 import re
 import types
+import ssl
 import sys
 
 import youtube_dl.extractor
@@ -244,3 +245,12 @@ def expect_warnings(ydl, warnings_re):
             real_warning(w)
 
     ydl.report_warning = _report_warning
+
+
+def http_server_port(httpd):
+    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
+        # In Jython SSLSocket is not a subclass of socket.socket
+        sock = httpd.socket.sock
+    else:
+        sock = httpd.socket
+    return sock.getsockname()[1]
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 4833396a5..06be72616 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -9,11 +9,30 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL, expect_dict, expect_value
-from youtube_dl.compat import compat_etree_fromstring
+from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
+from youtube_dl.compat import compat_etree_fromstring, compat_http_server
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
 from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
+import threading
+
+
+TEAPOT_RESPONSE_STATUS = 418
+TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
+
+
+class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def do_GET(self):
+        if self.path == '/teapot':
+            self.send_response(TEAPOT_RESPONSE_STATUS)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.end_headers()
+            self.wfile.write(TEAPOT_RESPONSE_BODY.encode())
+        else:
+            assert False
 
 
 class TestIE(InfoExtractor):
@@ -743,6 +762,25 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 for i in range(len(entries)):
                     expect_dict(self, entries[i], expected_entries[i])
 
+    def test_response_with_expected_status_returns_content(self):
+        # Checks for mitigations against the effects of
+        # <https://bugs.python.org/issue15002> that affect Python 3.4.1+, which
+        # manifest as `_download_webpage`, `_download_xml`, `_download_json`,
+        # or the underlying `_download_webpage_handle` returning no content
+        # when a response matches `expected_status`.
+
+        httpd = compat_http_server.HTTPServer(
+            ('127.0.0.1', 0), InfoExtractorTestRequestHandler)
+        port = http_server_port(httpd)
+        server_thread = threading.Thread(target=httpd.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+
+        (content, urlh) = self.ie._download_webpage_handle(
+            'http://127.0.0.1:%d/teapot' % port, None,
+            expected_status=TEAPOT_RESPONSE_STATUS)
+        self.assertEqual(content, TEAPOT_RESPONSE_BODY)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py
index 5cf2bf1a5..750472281 100644
--- a/test/test_downloader_http.py
+++ b/test/test_downloader_http.py
@@ -9,26 +9,16 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import try_rm
+from test.helper import http_server_port, try_rm
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_http_server
 from youtube_dl.downloader.http import HttpFD
 from youtube_dl.utils import encodeFilename
-import ssl
 import threading
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def http_server_port(httpd):
-    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
-        # In Jython SSLSocket is not a subclass of socket.socket
-        sock = httpd.socket.sock
-    else:
-        sock = httpd.socket
-    return sock.getsockname()[1]
-
-
 TEST_SIZE = 10 * 1024
 
 
diff --git a/test/test_http.py b/test/test_http.py
index 409fec9c8..3ee0a5dda 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,6 +8,7 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from test.helper import http_server_port
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_http_server, compat_urllib_request
 import ssl
@@ -16,15 +17,6 @@ import threading
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def http_server_port(httpd):
-    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
-        # In Jython SSLSocket is not a subclass of socket.socket
-        sock = httpd.socket.sock
-    else:
-        sock = httpd.socket
-    return sock.getsockname()[1]
-
-
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8452125c8..e5f8136fc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -606,6 +606,11 @@ class InfoExtractor(object):
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
             if isinstance(err, compat_urllib_error.HTTPError):
                 if self.__can_accept_status_code(err, expected_status):
+                    # Retain reference to error to prevent file object from
+                    # being closed before it can be read. Works around the
+                    # effects of <https://bugs.python.org/issue15002>
+                    # introduced in Python 3.4.1.
+                    err.fp._error = err
                     return err.fp
 
             if errnote is False:

From da56fb631fb6389849f2f021bdd047aa1c55dc0a Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Tue, 2 Oct 2018 14:49:01 +0200
Subject: [PATCH 147/510] [azmedien] Adopt to major site redesign (closes
 #17745)

---
 youtube_dl/extractor/azmedien.py   | 222 +++++++----------------------
 youtube_dl/extractor/extractors.py |   6 +-
 2 files changed, 53 insertions(+), 175 deletions(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index 68f26e2ca..9d606ee67 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -1,19 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
 from .kaltura import KalturaIE
-from ..utils import (
-    get_element_by_class,
-    get_element_by_id,
-    strip_or_none,
-    urljoin,
-)
 
 
 class AZMedienBaseIE(InfoExtractor):
+    _PARTNER_ID = '1719221'
+
     def _kaltura_video(self, partner_id, entry_id):
         return self.url_result(
             'kaltura:%s:%s' % (partner_id, entry_id), ie=KalturaIE.ie_key(),
@@ -25,189 +22,74 @@ class AZMedienIE(AZMedienBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?
-                        (?:
+                        (?P<host>
                             telezueri\.ch|
                             telebaern\.tv|
                             telem1\.ch
                         )/
-                        [0-9]+-show-[^/\#]+
-                        (?:
-                            /[0-9]+-episode-[^/\#]+
-                            (?:
-                                /[0-9]+-segment-(?:[^/\#]+\#)?|
-                                \#
-                            )|
-                            \#
+                        [^/]+/
+                        (?P<id>
+                            [^/]+-(?P<article_id>\d+)
                         )
-                        (?P<id>[^\#]+)
+                        (?:
+                            \#video=
+                            (?P<kaltura_id>
+                                [_0-9a-z]+
+                            )
+                        )?
                     '''
 
     _TESTS = [{
-        # URL with 'segment'
-        'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
+        'url': 'https://www.telezueri.ch/sonntalk/bundesrats-vakanzen-eu-rahmenabkommen-133214569',
         'info_dict': {
-            'id': '1_2444peh4',
+            'id': '1_anruz3wy',
             'ext': 'mp4',
-            'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
-            'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
-            'uploader_id': 'TeleZ?ri',
-            'upload_date': '20161218',
-            'timestamp': 1482084490,
+            'title': 'Bundesrats-Vakanzen / EU-Rahmenabkommen',
+            'description': 'md5:dd9f96751ec9c35e409a698a328402f3',
+            'uploader_id': 'TVOnline',
+            'upload_date': '20180930',
+            'timestamp': 1538328802,
         },
         'params': {
             'skip_download': True,
         },
     }, {
-        # URL with 'segment' and fragment:
-        'url': 'http://www.telebaern.tv/118-show-news/14240-episode-dienstag-17-januar-2017/33666-segment-achtung-gefahr#zu-wenig-pflegerinnen-und-pfleger',
-        'only_matching': True
-    }, {
-        # URL with 'episode' and fragment:
-        'url': 'http://www.telem1.ch/47-show-sonntalk/13986-episode-soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz#soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz',
-        'only_matching': True
-    }, {
-        # URL with 'show' and fragment:
-        'url': 'http://www.telezueri.ch/66-show-sonntalk#burka-plakate-trump-putin-china-besuch',
+        'url': 'https://www.telebaern.tv/telebaern-news/montag-1-oktober-2018-ganze-sendung-133531189#video=0_7xjo9lf1',
         'only_matching': True
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        entry_id = mobj.group('kaltura_id')
 
-        webpage = self._download_webpage(url, video_id)
+        if not entry_id:
+            webpage = self._download_webpage(url, video_id)
+            api_path = self._search_regex(
+                r'["\']apiPath["\']\s*:\s*["\']([^"^\']+)["\']',
+                webpage, 'api path')
+            api_url = 'https://www.%s%s' % (mobj.group('host'), api_path)
+            payload = {
+                'query': '''query VideoContext($articleId: ID!) {
+                    article: node(id: $articleId) {
+                      ... on Article {
+                        mainAssetRelation {
+                          asset {
+                            ... on VideoAsset {
+                              kalturaId
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }''',
+                'variables': {'articleId': 'Article:%s' % mobj.group('article_id')},
+            }
+            json_data = self._download_json(
+                api_url, video_id, headers={
+                    'Content-Type': 'application/json',
+                },
+                data=json.dumps(payload).encode())
+            entry_id = json_data['data']['article']['mainAssetRelation']['asset']['kalturaId']
 
-        partner_id = self._search_regex(
-            r'<script[^>]+src=["\'](?:https?:)?//(?:[^/]+\.)?kaltura\.com(?:/[^/]+)*/(?:p|partner_id)/([0-9]+)',
-            webpage, 'kaltura partner id')
-        entry_id = self._html_search_regex(
-            r'<a[^>]+data-id=(["\'])(?P<id>(?:(?!\1).)+)\1[^>]+data-slug=["\']%s'
-            % re.escape(video_id), webpage, 'kaltura entry id', group='id')
-
-        return self._kaltura_video(partner_id, entry_id)
-
-
-class AZMedienPlaylistIE(AZMedienBaseIE):
-    IE_DESC = 'AZ Medien playlists'
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        (?:
-                            telezueri\.ch|
-                            telebaern\.tv|
-                            telem1\.ch
-                        )/
-                        (?P<id>[0-9]+-
-                            (?:
-                                show|
-                                topic|
-                                themen
-                            )-[^/\#]+
-                            (?:
-                                /[0-9]+-episode-[^/\#]+
-                            )?
-                        )$
-                    '''
-
-    _TESTS = [{
-        # URL with 'episode'
-        'url': 'http://www.telebaern.tv/118-show-news/13735-episode-donnerstag-15-dezember-2016',
-        'info_dict': {
-            'id': '118-show-news/13735-episode-donnerstag-15-dezember-2016',
-            'title': 'News - Donnerstag, 15. Dezember 2016',
-        },
-        'playlist_count': 9,
-    }, {
-        # URL with 'themen'
-        'url': 'http://www.telem1.ch/258-themen-tele-m1-classics',
-        'info_dict': {
-            'id': '258-themen-tele-m1-classics',
-            'title': 'Tele M1 Classics',
-        },
-        'playlist_mincount': 15,
-    }, {
-        # URL with 'topic', contains nested playlists
-        'url': 'http://www.telezueri.ch/219-topic-aera-trump-hat-offiziell-begonnen',
-        'only_matching': True,
-    }, {
-        # URL with 'show' only
-        'url': 'http://www.telezueri.ch/86-show-talktaeglich',
-        'only_matching': True
-    }]
-
-    def _real_extract(self, url):
-        show_id = self._match_id(url)
-        webpage = self._download_webpage(url, show_id)
-
-        entries = []
-
-        partner_id = self._search_regex(
-            r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
-            webpage, 'kaltura partner id', default=None)
-
-        if partner_id:
-            entries = [
-                self._kaltura_video(partner_id, m.group('id'))
-                for m in re.finditer(
-                    r'data-id=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage)]
-
-        if not entries:
-            entries = [
-                self.url_result(m.group('url'), ie=AZMedienIE.ie_key())
-                for m in re.finditer(
-                    r'<a[^>]+data-real=(["\'])(?P<url>http.+?)\1', webpage)]
-
-        if not entries:
-            entries = [
-                # May contain nested playlists (e.g. [1]) thus no explicit
-                # ie_key
-                # 1. http://www.telezueri.ch/219-topic-aera-trump-hat-offiziell-begonnen)
-                self.url_result(urljoin(url, m.group('url')))
-                for m in re.finditer(
-                    r'<a[^>]+name=[^>]+href=(["\'])(?P<url>/.+?)\1', webpage)]
-
-        title = self._search_regex(
-            r'episodeShareTitle\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
-            webpage, 'title',
-            default=strip_or_none(get_element_by_id(
-                'video-title', webpage)), group='title')
-
-        return self.playlist_result(entries, show_id, title)
-
-
-class AZMedienShowPlaylistIE(AZMedienBaseIE):
-    IE_DESC = 'AZ Medien show playlists'
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        (?:
-                            telezueri\.ch|
-                            telebaern\.tv|
-                            telem1\.ch
-                        )/
-                        (?:
-                            all-episodes|
-                            alle-episoden
-                        )/
-                        (?P<id>[^/?#&]+)
-                    '''
-
-    _TEST = {
-        'url': 'http://www.telezueri.ch/all-episodes/astrotalk',
-        'info_dict': {
-            'id': 'astrotalk',
-            'title': 'TeleZüri: AstroTalk - alle episoden',
-            'description': 'md5:4c0f7e7d741d906004266e295ceb4a26',
-        },
-        'playlist_mincount': 13,
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-        episodes = get_element_by_class('search-mobile-box', webpage)
-        entries = [self.url_result(
-            urljoin(url, m.group('url'))) for m in re.finditer(
-                r'<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1', episodes)]
-        title = self._og_search_title(webpage, fatal=False)
-        description = self._og_search_description(webpage)
-        return self.playlist_result(entries, playlist_id, title, description)
+        return self._kaltura_video(self._PARTNER_ID, entry_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b41cd65d7..9b68c9efe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -88,11 +88,7 @@ from .awaan import (
     AWAANLiveIE,
     AWAANSeasonIE,
 )
-from .azmedien import (
-    AZMedienIE,
-    AZMedienPlaylistIE,
-    AZMedienShowPlaylistIE,
-)
+from .azmedien import AZMedienIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE

From 573531dcfb6869afa143761faf4ebc6ab405f808 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Nov 2018 01:32:29 +0700
Subject: [PATCH 148/510] [azmedien] Simplify (closes #17746)

---
 youtube_dl/extractor/azmedien.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index 9d606ee67..a57a5f114 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -8,16 +8,7 @@ from .common import InfoExtractor
 from .kaltura import KalturaIE
 
 
-class AZMedienBaseIE(InfoExtractor):
-    _PARTNER_ID = '1719221'
-
-    def _kaltura_video(self, partner_id, entry_id):
-        return self.url_result(
-            'kaltura:%s:%s' % (partner_id, entry_id), ie=KalturaIE.ie_key(),
-            video_id=entry_id)
-
-
-class AZMedienIE(AZMedienBaseIE):
+class AZMedienIE(InfoExtractor):
     IE_DESC = 'AZ Medien videos'
     _VALID_URL = r'''(?x)
                     https?://
@@ -58,9 +49,11 @@ class AZMedienIE(AZMedienBaseIE):
         'only_matching': True
     }]
 
+    _PARTNER_ID = '1719221'
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
         mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
         entry_id = mobj.group('kaltura_id')
 
         if not entry_id:
@@ -92,4 +85,6 @@ class AZMedienIE(AZMedienBaseIE):
                 data=json.dumps(payload).encode())
             entry_id = json_data['data']['article']['mainAssetRelation']['asset']['kalturaId']
 
-        return self._kaltura_video(self._PARTNER_ID, entry_id)
+        return self.url_result(
+            'kaltura:%s:%s' % (self._PARTNER_ID, entry_id),
+            ie=KalturaIE.ie_key(), video_id=entry_id)

From faac1c1f70425ddd60ff39d3b6a2b34c7941463b Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Tue, 1 May 2018 05:36:03 +0200
Subject: [PATCH 149/510] [ehftv] Add extractor (closes #15408)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/laola1tv.py   | 115 ++++++++++++++++++-----------
 2 files changed, 73 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9b68c9efe..e5488cce4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -539,6 +539,7 @@ from .la7 import LA7IE
 from .laola1tv import (
     Laola1TvEmbedIE,
     Laola1TvIE,
+    EHFTVIE,
     ITTFIE,
 )
 from .lci import LCIIE
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index c7f813370..d985bd3ca 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -119,9 +120,59 @@ class Laola1TvEmbedIE(InfoExtractor):
         }
 
 
-class Laola1TvIE(Laola1TvEmbedIE):
+class Laola1TvBaseIE(Laola1TvEmbedIE):
+    def _extract_video(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        if 'Dieser Livestream ist bereits beendet.' in webpage:
+            raise ExtractorError('This live stream has already finished.', expected=True)
+
+        conf = self._parse_json(self._search_regex(
+            r'(?s)conf\s*=\s*({.+?});', webpage, 'conf'),
+            display_id,
+            transform_source=lambda s: js_to_json(re.sub(r'shareurl:.+,', '', s)))
+        video_id = conf['videoid']
+
+        config = self._download_json(conf['configUrl'], video_id, query={
+            'videoid': video_id,
+            'partnerid': conf['partnerid'],
+            'language': conf.get('language', ''),
+            'portal': conf.get('portalid', ''),
+        })
+        error = config.get('error')
+        if error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+        video_data = config['video']
+        title = video_data['title']
+        is_live = video_data.get('isLivestream') and video_data.get('isLive')
+        meta = video_data.get('metaInformation')
+        sports = meta.get('sports')
+        categories = sports.split(',') if sports else []
+
+        token_url = self._extract_token_url(
+            video_data['streamAccess'], video_id,
+            video_data['abo']['required'])
+
+        formats = self._extract_formats(token_url, video_id)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': self._live_title(title) if is_live else title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('image'),
+            'categories': categories,
+            'formats': formats,
+            'is_live': is_live,
+        }
+
+
+class Laola1TvIE(Laola1TvBaseIE):
     IE_NAME = 'laola1tv'
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/[a-z]+-[a-z]+/[^/]+/(?P<id>[^/?#&]+)'
+
     _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
@@ -169,52 +220,30 @@ class Laola1TvIE(Laola1TvEmbedIE):
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        return self._extract_video(url)
 
-        webpage = self._download_webpage(url, display_id)
 
-        if 'Dieser Livestream ist bereits beendet.' in webpage:
-            raise ExtractorError('This live stream has already finished.', expected=True)
+class EHFTVIE(Laola1TvBaseIE):
+    IE_NAME = 'ehftv'
+    _VALID_URL = r'https?://(?:www\.)?ehftv\.com/[a-z]+(-[a-z]+)?/[^/]+/(?P<id>[^/?#&]+)'
 
-        conf = self._parse_json(self._search_regex(
-            r'(?s)conf\s*=\s*({.+?});', webpage, 'conf'),
-            display_id, js_to_json)
+    _TESTS = [{
+        'url': 'https://www.ehftv.com/int/video/paris-saint-germain-handball-pge-vive-kielce/1166761',
+        'info_dict': {
+            'id': '1166761',
+            'display_id': 'paris-saint-germain-handball-pge-vive-kielce',
+            'ext': 'mp4',
+            'title': 'Paris Saint-Germain Handball - PGE Vive Kielce',
+            'is_live': False,
+            'categories': ['Handball'],
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
-        video_id = conf['videoid']
-
-        config = self._download_json(conf['configUrl'], video_id, query={
-            'videoid': video_id,
-            'partnerid': conf['partnerid'],
-            'language': conf.get('language', ''),
-            'portal': conf.get('portalid', ''),
-        })
-        error = config.get('error')
-        if error:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
-
-        video_data = config['video']
-        title = video_data['title']
-        is_live = video_data.get('isLivestream') and video_data.get('isLive')
-        meta = video_data.get('metaInformation')
-        sports = meta.get('sports')
-        categories = sports.split(',') if sports else []
-
-        token_url = self._extract_token_url(
-            video_data['streamAccess'], video_id,
-            video_data['abo']['required'])
-
-        formats = self._extract_formats(token_url, video_id)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': self._live_title(title) if is_live else title,
-            'description': video_data.get('description'),
-            'thumbnail': video_data.get('image'),
-            'categories': categories,
-            'formats': formats,
-            'is_live': is_live,
-        }
+    def _real_extract(self, url):
+        return self._extract_video(url)
 
 
 class ITTFIE(InfoExtractor):

From 6895ea4d3f9d44048fa59800a06ab8177a24bd1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Nov 2018 02:44:35 +0700
Subject: [PATCH 150/510] [laola1tv:embed] Set correct stream access URL scheme
 (closes #16341)

---
 youtube_dl/extractor/laola1tv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index d985bd3ca..fa217365a 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -33,7 +33,8 @@ class Laola1TvEmbedIE(InfoExtractor):
 
     def _extract_token_url(self, stream_access_url, video_id, data):
         return self._download_json(
-            stream_access_url, video_id, headers={
+            self._proto_relative_url(stream_access_url, 'https:'), video_id,
+            headers={
                 'Content-Type': 'application/json',
             }, data=json.dumps(data).encode())['data']['stream-access'][0]
 
@@ -225,7 +226,7 @@ class Laola1TvIE(Laola1TvBaseIE):
 
 class EHFTVIE(Laola1TvBaseIE):
     IE_NAME = 'ehftv'
-    _VALID_URL = r'https?://(?:www\.)?ehftv\.com/[a-z]+(-[a-z]+)?/[^/]+/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?ehftv\.com/[a-z]+(?:-[a-z]+)?/[^/]+/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'https://www.ehftv.com/int/video/paris-saint-germain-handball-pge-vive-kielce/1166761',

From a085410936a5e0c90a8eb3059a5cd9e0703848bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Nov 2018 02:56:14 +0700
Subject: [PATCH 151/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 57dbde12d..05857596a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,25 @@
+version <unreleased>
+
+Core
+* [extractor/common] Ensure response handle is not prematurely closed before
+  it can be read if it matches expected_status (#17195, #17846, #17447)
+
+Extractors
+* [laola1tv:embed] Set correct stream access URL scheme (#16341)
++ [ehftv] Add support for ehftv.com (#15408)
+* [azmedien] Adopt to major site redesign (#17745, #17746)
++ [twitcasting] Add support for twitcasting.tv (#17981)
+* [orf:tvthek] Fix extraction (#17737, #17956, #18024)
++ [openload] Add support for oload.fun (#18045)
+* [njpwworld] Fix authentication (#17427)
++ [linkedin:learning] Add support for linkedin.com/learning (#13545)
+* [theplatform] Improve error detection (#13222)
+* [cnbc] Simplify extraction (#14280, #17110)
++ [cbnc] Add support for new URL schema (#14193)
+* [aparat] Improve extraction and extract more metadata (#17445, #18008)
+* [aparat] Fix extraction
+
+
 version 2018.10.29
 
 Core

From 38c32dbf19d0168295d02b0afaed9227fed46338 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Nov 2018 02:57:48 +0700
Subject: [PATCH 152/510] release 2018.11.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 7 +++++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index aefed163a..eb8cef8ef 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.10.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.03*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.10.29
+[debug] youtube-dl version 2018.11.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 05857596a..11e1ba333 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.11.03
 
 Core
 * [extractor/common] Ensure response handle is not prematurely closed before
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e5a6879bc..24c3254c3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -84,8 +84,6 @@
  - **awaan:season**
  - **awaan:video**
  - **AZMedien**: AZ Medien videos
- - **AZMedienPlaylist**: AZ Medien playlists
- - **AZMedienShowPlaylist**: AZ Medien show playlists
  - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
@@ -178,6 +176,7 @@
  - **Clyp**
  - **cmt.com**
  - **CNBC**
+ - **CNBCVideo**
  - **CNN**
  - **CNNArticle**
  - **CNNBlogs**
@@ -251,6 +250,7 @@
  - **EchoMsk**
  - **egghead:course**: egghead.io course
  - **egghead:lesson**: egghead.io lesson
+ - **ehftv**
  - **eHow**
  - **EinsUndEinsTV**
  - **Einthusan**
@@ -445,6 +445,8 @@
  - **limelight:channel**
  - **limelight:channel_list**
  - **LineTV**
+ - **linkedin:learning**
+ - **linkedin:learning:course**
  - **LiTV**
  - **LiveLeak**
  - **LiveLeakEmbed**
@@ -930,6 +932,7 @@
  - **TVPlayer**
  - **TVPlayHome**
  - **Tweakers**
+ - **TwitCasting**
  - **twitch:chapter**
  - **twitch:clips**
  - **twitch:profile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ae9a77966..90de01214 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.10.29'
+__version__ = '2018.11.03'

From dbdaaa231add0a8d1fa8138c448ccb344f585894 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Nov 2018 06:26:16 +0700
Subject: [PATCH 153/510] [youtube] Add fallback metadata extraction from
 videoDetails (closes #18052)

---
 youtube_dl/extractor/youtube.py | 34 ++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 78203ef84..abadfa545 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -41,6 +41,7 @@ from ..utils import (
     remove_quotes,
     remove_start,
     smuggle_url,
+    str_or_none,
     str_to_int,
     try_get,
     unescapeHTML,
@@ -501,6 +502,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
                 'duration': 10,
+                'view_count': int,
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
@@ -583,6 +585,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
                 'duration': 10,
+                'view_count': int,
                 'like_count': int,
                 'dislike_count': int,
             },
@@ -1538,6 +1541,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         def extract_view_count(v_info):
             return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
 
+        player_response = {}
+
         # Get video info
         embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
@@ -1580,6 +1585,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
                     is_live = True
                 sts = ytplayer_config.get('sts')
+                if not player_response:
+                    pl_response = str_or_none(args.get('player_response'))
+                    if pl_response:
+                        pl_response = self._parse_json(pl_response, video_id, fatal=False)
+                        if isinstance(pl_response, dict):
+                            player_response = pl_response
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1608,6 +1619,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info_webpage:
                         continue
                     get_video_info = compat_parse_qs(video_info_webpage)
+                    if not player_response:
+                        pl_response = get_video_info.get('player_response', [None])[0]
+                        if isinstance(pl_response, dict):
+                            player_response = pl_response
                     add_dash_mpd(get_video_info)
                     if view_count is None:
                         view_count = extract_view_count(get_video_info)
@@ -1653,9 +1668,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     '"token" parameter not in video info for unknown reason',
                     video_id=video_id)
 
+        video_details = try_get(
+            player_response, lambda x: x['videoDetails'], dict) or {}
+
         # title
         if 'title' in video_info:
             video_title = video_info['title'][0]
+        elif 'title' in player_response:
+            video_title = video_details['title']
         else:
             self._downloader.report_warning('Unable to extract video title')
             video_title = '_'
@@ -1718,6 +1738,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         if view_count is None:
             view_count = extract_view_count(video_info)
+        if view_count is None and video_details:
+            view_count = int_or_none(video_details.get('viewCount'))
 
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
@@ -1898,7 +1920,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 
         # uploader
-        video_uploader = try_get(video_info, lambda x: x['author'][0], compat_str)
+        video_uploader = try_get(
+            video_info, lambda x: x['author'][0],
+            compat_str) or str_or_none(video_details.get('author'))
         if video_uploader:
             video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
         else:
@@ -2011,12 +2035,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         like_count = _extract_count('like')
         dislike_count = _extract_count('dislike')
 
+        if view_count is None:
+            view_count = str_to_int(self._search_regex(
+                r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
+                'view count', default=None))
+
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, video_webpage)
         automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
 
         video_duration = try_get(
             video_info, lambda x: int_or_none(x['length_seconds'][0]))
+        if not video_duration:
+            video_duration = int_or_none(video_details.get('lengthSeconds'))
         if not video_duration:
             video_duration = parse_duration(self._html_search_meta(
                 'duration', video_webpage, 'video duration'))
@@ -2244,6 +2275,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
             'categories': ['People & Blogs'],
             'tags': list,
+            'view_count': int,
             'like_count': int,
             'dislike_count': int,
         },

From 22e07ce502275fbede32d212eacdaeabee22fe4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Nov 2018 00:11:36 +0700
Subject: [PATCH 154/510] [README.md] Improve documentation on safe metadata
 extraction and add more examples

---
 README.md | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fdd115c9b..35c3de512 100644
--- a/README.md
+++ b/README.md
@@ -1168,7 +1168,28 @@ title = self._search_regex(
 
 ### Use safe conversion functions
 
-Wrap all extracted numeric data into safe functions from `utils`: `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+
+Use `url_or_none` for safe URL processing.
+
+Use `try_get` for safe metadata extraction from parsed JSON.
+
+Explore [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
+
+#### More examples
+
+##### Safely extract optional description from parsed JSON
+```python
+description = try_get(response, lambda x: x['result']['video'][0]['summary'], compat_str)
+```
+
+##### Safely extract more optional metadata
+```python
+video = try_get(response, lambda x: x['result']['video'][0], dict) or {}
+description = video.get('summary')
+duration = float_or_none(video.get('durationMs'), scale=1000)
+view_count = int_or_none(video.get('views'))
+```
 
 # EMBEDDING YOUTUBE-DL
 

From 16d896b2a74e2b9989fc0483728f8009876fc4cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Nov 2018 15:52:46 +0700
Subject: [PATCH 155/510] [zattoo] Arrange API hosts for derived extractors
 (closes #18035)

---
 youtube_dl/extractor/zattoo.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/zattoo.py b/youtube_dl/extractor/zattoo.py
index bbe0aecb6..cb1bac3a3 100644
--- a/youtube_dl/extractor/zattoo.py
+++ b/youtube_dl/extractor/zattoo.py
@@ -22,7 +22,7 @@ class ZattooPlatformBaseIE(InfoExtractor):
     _power_guide_hash = None
 
     def _host_url(self):
-        return 'https://%s' % self._HOST
+        return 'https://%s' % (self._API_HOST if hasattr(self, '_API_HOST') else self._HOST)
 
     def _login(self):
         username, password = self._get_login_info()
@@ -286,6 +286,7 @@ class ZattooLiveIE(ZattooBaseIE):
 class NetPlusIE(ZattooIE):
     _NETRC_MACHINE = 'netplus'
     _HOST = 'netplus.tv'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
@@ -300,7 +301,7 @@ class MNetTVIE(ZattooIE):
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
-        'url': 'https://www.tvplus.m-net.de/watch/abc/123-abc',
+        'url': 'https://tvplus.m-net.de/watch/abc/123-abc',
         'only_matching': True,
     }]
 
@@ -311,7 +312,7 @@ class WalyTVIE(ZattooIE):
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
-        'url': 'https://www.player.waly.tv/watch/abc/123-abc',
+        'url': 'https://player.waly.tv/watch/abc/123-abc',
         'only_matching': True,
     }]
 
@@ -319,6 +320,7 @@ class WalyTVIE(ZattooIE):
 class BBVTVIE(ZattooIE):
     _NETRC_MACHINE = 'bbvtv'
     _HOST = 'bbv-tv.net'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
@@ -330,6 +332,7 @@ class BBVTVIE(ZattooIE):
 class VTXTVIE(ZattooIE):
     _NETRC_MACHINE = 'vtxtv'
     _HOST = 'vtxtv.ch'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
@@ -341,6 +344,7 @@ class VTXTVIE(ZattooIE):
 class MyVisionTVIE(ZattooIE):
     _NETRC_MACHINE = 'myvisiontv'
     _HOST = 'myvisiontv.ch'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
@@ -355,7 +359,7 @@ class GlattvisionTVIE(ZattooIE):
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
-        'url': 'https://www.iptv.glattvision.ch/watch/abc/123-abc',
+        'url': 'https://iptv.glattvision.ch/watch/abc/123-abc',
         'only_matching': True,
     }]
 
@@ -363,6 +367,7 @@ class GlattvisionTVIE(ZattooIE):
 class SAKTVIE(ZattooIE):
     _NETRC_MACHINE = 'saktv'
     _HOST = 'saktv.ch'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
@@ -377,7 +382,7 @@ class EWETVIE(ZattooIE):
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
-        'url': 'https://www.tvonline.ewe.de/watch/abc/123-abc',
+        'url': 'https://tvonline.ewe.de/watch/abc/123-abc',
         'only_matching': True,
     }]
 
@@ -385,6 +390,7 @@ class EWETVIE(ZattooIE):
 class QuantumTVIE(ZattooIE):
     _NETRC_MACHINE = 'quantumtv'
     _HOST = 'quantum-tv.com'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
@@ -399,7 +405,7 @@ class OsnatelTVIE(ZattooIE):
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{
-        'url': 'https://www.onlinetv.osnatel.de/watch/abc/123-abc',
+        'url': 'https://tvonline.osnatel.de/watch/abc/123-abc',
         'only_matching': True,
     }]
 
@@ -407,6 +413,7 @@ class OsnatelTVIE(ZattooIE):
 class EinsUndEinsTVIE(ZattooIE):
     _NETRC_MACHINE = '1und1tv'
     _HOST = '1und1.tv'
+    _API_HOST = 'www.%s' % _HOST
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{

From 2004e2210bc74aa950feba0f22df4d5a8980b3e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Nov 2018 17:09:57 +0700
Subject: [PATCH 156/510] [osnateltv] Update host

---
 youtube_dl/extractor/zattoo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zattoo.py b/youtube_dl/extractor/zattoo.py
index cb1bac3a3..896276301 100644
--- a/youtube_dl/extractor/zattoo.py
+++ b/youtube_dl/extractor/zattoo.py
@@ -401,7 +401,7 @@ class QuantumTVIE(ZattooIE):
 
 class OsnatelTVIE(ZattooIE):
     _NETRC_MACHINE = 'osnateltv'
-    _HOST = 'onlinetv.osnatel.de'
+    _HOST = 'tvonline.osnatel.de'
     _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
 
     _TESTS = [{

From c0345b825f8758571a8de871ab9349c46b062fc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Nov 2018 19:08:39 +0700
Subject: [PATCH 157/510] [youtube:playlist] Add support for invidio.us (closes
 #18077)

---
 youtube_dl/extractor/youtube.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index abadfa545..6ab2db274 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2162,7 +2162,11 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
                         (?:https?://)?
                         (?:\w+\.)?
                         (?:
-                            youtube\.com/
+                            (?:
+                                youtube\.com|
+                                invidio\.us
+                            )
+                            /
                             (?:
                                (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
                                \? (?:.*?[&;])*? (?:p|a|list)=
@@ -2314,6 +2318,9 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         # music album playlist
         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
         'only_matching': True,
+    }, {
+        'url': 'https://invidio.us/playlist?list=PLDIoUOhQQPlXr63I_vwF9GD8sAKh77dWU',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):

From 432cd4841023091811db46cd82c188698a386841 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Nov 2018 23:29:42 +0700
Subject: [PATCH 158/510] [cliphinter] Fix extraction (closes #18083)

---
 youtube_dl/extractor/cliphunter.py | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index ab651d1c8..f2ca7a337 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -1,19 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-_translation_table = {
-    'a': 'h', 'd': 'e', 'e': 'v', 'f': 'o', 'g': 'f', 'i': 'd', 'l': 'n',
-    'm': 'a', 'n': 'm', 'p': 'u', 'q': 't', 'r': 's', 'v': 'p', 'x': 'r',
-    'y': 'l', 'z': 'i',
-    '$': ':', '&': '.', '(': '=', '^': '&', '=': '/',
-}
-
-
-def _decode(s):
-    return ''.join(_translation_table.get(c, c) for c in s)
+from ..utils import (
+    int_or_none,
+    url_or_none,
+)
 
 
 class CliphunterIE(InfoExtractor):
@@ -60,14 +51,14 @@ class CliphunterIE(InfoExtractor):
 
         formats = []
         for format_id, f in gexo_files.items():
-            video_url = f.get('url')
+            video_url = url_or_none(f.get('url'))
             if not video_url:
                 continue
             fmt = f.get('fmt')
             height = f.get('h')
             format_id = '%s_%sp' % (fmt, height) if fmt and height else format_id
             formats.append({
-                'url': _decode(video_url),
+                'url': video_url,
                 'format_id': format_id,
                 'width': int_or_none(f.get('w')),
                 'height': int_or_none(height),

From 0df514f07e23ce70cccec045b4e71bdec151fcc7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 6 Nov 2018 21:22:00 +0100
Subject: [PATCH 159/510] [facebook] fix tahoe request(closes #17171)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 97cfe0fc3..74954049d 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -57,7 +57,7 @@ class FacebookIE(InfoExtractor):
     _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
 
     _VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'
-    _VIDEO_PAGE_TAHOE_TEMPLATE = 'https://www.facebook.com/video/tahoe/async/%s/?chain=true&isvideo=true'
+    _VIDEO_PAGE_TAHOE_TEMPLATE = 'https://www.facebook.com/video/tahoe/async/%s/?chain=true&isvideo=true&payloadtype=primary'
 
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',

From 2511eee215c2a66020ae927c86face826f48ba8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Nov 2018 09:55:59 +0700
Subject: [PATCH 160/510] [youtube] Add another JS signature function name
 regex (closes #18091, closes #18093, closes #18094)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6ab2db274..3f49f3889 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1192,7 +1192,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*c\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\('),
+             r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
         jsi = JSInterpreter(jscode)

From f81d44aab6d8ee01024a637cb80374251737872e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Nov 2018 09:58:08 +0700
Subject: [PATCH 161/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 11e1ba333..920a4855a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+version <unreleased>
+
+Extractors
++ [youtube] Add another JS signature function name regex (#18091, #18093,
+  #18094)
+* [facebook] Fix tahoe request (#17171)
+* [cliphunter] Fix extraction (#18083)
++ [youtube:playlist] Add support for invidio.us (#18077)
+* [zattoo] Arrange API hosts for derived extractors (#18035)
++ [youtube] Add fallback metadata extraction from videoDetails (#18052)
+
+
 version 2018.11.03
 
 Core

From 532782ade1dab884606dbbd82081ed7ab9c52a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Nov 2018 01:38:25 +0700
Subject: [PATCH 162/510] release 2018.11.07

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 CONTRIBUTING.md           | 23 ++++++++++++++++++++++-
 ChangeLog                 |  2 +-
 youtube_dl/version.py     |  2 +-
 4 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index eb8cef8ef..7607e0e03 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.03*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.11.03
+[debug] youtube-dl version 2018.11.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 333acee80..bbcb78808 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -296,5 +296,26 @@ title = self._search_regex(
 
 ### Use safe conversion functions
 
-Wrap all extracted numeric data into safe functions from `utils`: `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+
+Use `url_or_none` for safe URL processing.
+
+Use `try_get` for safe metadata extraction from parsed JSON.
+
+Explore [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
+
+#### More examples
+
+##### Safely extract optional description from parsed JSON
+```python
+description = try_get(response, lambda x: x['result']['video'][0]['summary'], compat_str)
+```
+
+##### Safely extract more optional metadata
+```python
+video = try_get(response, lambda x: x['result']['video'][0], dict) or {}
+description = video.get('summary')
+duration = float_or_none(video.get('durationMs'), scale=1000)
+view_count = int_or_none(video.get('views'))
+```
 
diff --git a/ChangeLog b/ChangeLog
index 920a4855a..fa5de8b04 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.11.07
 
 Extractors
 + [youtube] Add another JS signature function name regex (#18091, #18093,
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 90de01214..7f32ad36c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.11.03'
+__version__ = '2018.11.07'

From cab26223bf480553d67840fc9f46aa9ff89ec29f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Nov 2018 15:22:59 +0700
Subject: [PATCH 163/510] [ruutu] Update API endpoint (closes #18138)

---
 youtube_dl/extractor/ruutu.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 9fa8688f8..f530f0083 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -65,7 +65,8 @@ class RuutuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video_xml = self._download_xml(
-            'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id, video_id)
+            'https://gatling.nelonenmedia.fi/media-xml-cache', video_id,
+            query={'id': video_id})
 
         formats = []
         processed_urls = []

From 96a91b15513af2121be1dd93871cc3769c06da3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Nov 2018 23:37:27 +0700
Subject: [PATCH 164/510] [vivo] Fix extraction (closes #18139)

---
 youtube_dl/extractor/shared.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index b2250afdd..931a0f70e 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -5,6 +5,7 @@ from ..compat import compat_b64decode
 from ..utils import (
     ExtractorError,
     int_or_none,
+    url_or_none,
     urlencode_postdata,
 )
 
@@ -86,9 +87,16 @@ class VivoIE(SharedBaseIE):
     }
 
     def _extract_video_url(self, webpage, video_id, *args):
+        def decode_url(encoded_url):
+            return compat_b64decode(encoded_url).decode('utf-8')
+
+        stream_url = url_or_none(decode_url(self._search_regex(
+            r'data-stream\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'stream url', default=None, group='url')))
+        if stream_url:
+            return stream_url
         return self._parse_json(
             self._search_regex(
                 r'InitializeStream\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1',
                 webpage, 'stream', group='url'),
-            video_id,
-            transform_source=lambda x: compat_b64decode(x).decode('utf-8'))[0]
+            video_id, transform_source=decode_url)[0]

From 83852e57bf2f96ba50418b1a888ae5a1836549cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Nov 2018 00:44:49 +0700
Subject: [PATCH 165/510] [zype] Add extractor (closes #18143)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/generic.py    | 20 +++++++++++
 youtube_dl/extractor/zype.py       | 57 ++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)
 create mode 100644 youtube_dl/extractor/zype.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e5488cce4..b2b00c86f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1478,3 +1478,4 @@ from .zattoo import (
 )
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import ZingMp3IE
+from .zype import ZypeIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 545e03371..59cf03faf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -114,6 +114,7 @@ from .apa import APAIE
 from .foxnews import FoxNewsIE
 from .viqeo import ViqeoIE
 from .expressen import ExpressenIE
+from .zype import ZypeIE
 
 
 class GenericIE(InfoExtractor):
@@ -2070,6 +2071,20 @@ class GenericIE(InfoExtractor):
             },
             'playlist_count': 6,
         },
+        {
+            # Zype embed
+            'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites',
+            'info_dict': {
+                'id': '5b400b834b32992a310622b9',
+                'ext': 'mp4',
+                'title': 'Smoky Barbecue Favorites',
+                'thumbnail': r're:^https?://.*\.jpe?g',
+            },
+            'add_ie': [ZypeIE.ie_key()],
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             # videojs embed
             'url': 'https://video.sibnet.ru/shell.php?videoid=3422904',
@@ -3129,6 +3144,11 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 expressen_urls, video_id, video_title, ie=ExpressenIE.ie_key())
 
+        zype_urls = ZypeIE._extract_urls(webpage)
+        if zype_urls:
+            return self.playlist_from_matches(
+                zype_urls, video_id, video_title, ie=ZypeIE.ie_key())
+
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:
diff --git a/youtube_dl/extractor/zype.py b/youtube_dl/extractor/zype.py
new file mode 100644
index 000000000..3b16e703b
--- /dev/null
+++ b/youtube_dl/extractor/zype.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ZypeIE(InfoExtractor):
+    _VALID_URL = r'https?://player\.zype\.com/embed/(?P<id>[\da-fA-F]+)\.js\?.*?api_key=[^&]+'
+    _TEST = {
+        'url': 'https://player.zype.com/embed/5b400b834b32992a310622b9.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ&autoplay=false&controls=true&da=false',
+        'md5': 'eaee31d474c76a955bdaba02a505c595',
+        'info_dict': {
+            'id': '5b400b834b32992a310622b9',
+            'ext': 'mp4',
+            'title': 'Smoky Barbecue Favorites',
+            'thumbnail': r're:^https?://.*\.jpe?g',
+        },
+    }
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return [
+            mobj.group('url')
+            for mobj in re.finditer(
+                r'<script[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//player\.zype\.com/embed/[\da-fA-F]+\.js\?.*?api_key=.+?)\1',
+                webpage)]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'video_title\s*[:=]\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+            'title', group='value')
+
+        m3u8_url = self._search_regex(
+            r'(["\'])(?P<url>(?:(?!\1).)+\.m3u8(?:(?!\1).)*)\1', webpage,
+            'm3u8 url', group='url')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+        self._sort_formats(formats)
+
+        thumbnail = self._search_regex(
+            r'poster\s*[:=]\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage, 'thumbnail',
+            default=False, group='url')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From f17a24a6df293370b94082c7feb6c447a3e7d8d9 Mon Sep 17 00:00:00 2001
From: Patrick Griffis <tingping@tingping.se>
Date: Mon, 21 May 2018 17:02:16 -0400
Subject: [PATCH 166/510] [picarto] Use API and add token support

This is just more reliable than trying to extract
it from the page itself.
---
 youtube_dl/extractor/picarto.py | 36 +++++++++------------------------
 1 file changed, 9 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/picarto.py b/youtube_dl/extractor/picarto.py
index 2366dfb34..27ee9643b 100644
--- a/youtube_dl/extractor/picarto.py
+++ b/youtube_dl/extractor/picarto.py
@@ -8,14 +8,13 @@ from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     js_to_json,
-    try_get,
     update_url_query,
     urlencode_postdata,
 )
 
 
 class PicartoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www.)?picarto\.tv/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www.)?picarto\.tv/(?P<id>[a-zA-Z0-9]+)(?:/(?P<token>[a-zA-Z0-9]+))?'
     _TEST = {
         'url': 'https://picarto.tv/Setz',
         'info_dict': {
@@ -34,19 +33,11 @@ class PicartoIE(InfoExtractor):
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
-        stream_page = self._download_webpage(url, channel_id)
+        metadata = self._download_json(
+            'https://api.picarto.tv/v1/channel/name/' + channel_id,
+            channel_id)
 
-        if '>This channel does not exist' in stream_page:
-            raise ExtractorError(
-                'Channel %s does not exist' % channel_id, expected=True)
-
-        player = self._parse_json(
-            self._search_regex(
-                r'(?s)playerSettings\[\d+\]\s*=\s*(\{.+?\}\s*\n)', stream_page,
-                'player settings'),
-            channel_id, transform_source=js_to_json)
-
-        if player.get('online') is False:
+        if metadata.get('online') is False:
             raise ExtractorError('Stream is offline', expected=True)
 
         cdn_data = self._download_json(
@@ -54,20 +45,13 @@ class PicartoIE(InfoExtractor):
             data=urlencode_postdata({'loadbalancinginfo': channel_id}),
             note='Downloading load balancing info')
 
-        def get_event(key):
-            return try_get(player, lambda x: x['event'][key], compat_str) or ''
-
+        token = self._VALID_URL_RE.match(url).group('token') or 'public'
         params = {
-            'token': player.get('token') or '',
-            'ticket': get_event('ticket'),
             'con': int(time.time() * 1000),
-            'type': get_event('ticket'),
-            'scope': get_event('scope'),
+            'token': token,
         }
 
         prefered_edge = cdn_data.get('preferedEdge')
-        default_tech = player.get('defaultTech')
-
         formats = []
 
         for edge in cdn_data['edges']:
@@ -81,8 +65,6 @@ class PicartoIE(InfoExtractor):
                 preference = 0
                 if edge_id == prefered_edge:
                     preference += 1
-                if tech_type == default_tech:
-                    preference += 1
                 format_id = []
                 if edge_id:
                     format_id.append(edge_id)
@@ -109,7 +91,7 @@ class PicartoIE(InfoExtractor):
                     continue
         self._sort_formats(formats)
 
-        mature = player.get('mature')
+        mature = metadata.get('adult')
         if mature is None:
             age_limit = None
         else:
@@ -119,7 +101,7 @@ class PicartoIE(InfoExtractor):
             'id': channel_id,
             'title': self._live_title(channel_id),
             'is_live': True,
-            'thumbnail': player.get('vodThumb'),
+            'thumbnail': metadata.get('thumbnails', {}).get('web'),
             'age_limit': age_limit,
             'formats': formats,
         }

From 730c0d12a06f349907481570f1f2890251f7a181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Nov 2018 16:08:54 +0700
Subject: [PATCH 167/510] [picarto] Extract more metadata (closes #16518)

---
 youtube_dl/extractor/picarto.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/picarto.py b/youtube_dl/extractor/picarto.py
index 27ee9643b..8099ef1d6 100644
--- a/youtube_dl/extractor/picarto.py
+++ b/youtube_dl/extractor/picarto.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import time
 
 from .common import InfoExtractor
@@ -8,6 +9,7 @@ from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     js_to_json,
+    try_get,
     update_url_query,
     urlencode_postdata,
 )
@@ -32,7 +34,9 @@ class PicartoIE(InfoExtractor):
         return False if PicartoVodIE.suitable(url) else super(PicartoIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        channel_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        channel_id = mobj.group('id')
+
         metadata = self._download_json(
             'https://api.picarto.tv/v1/channel/name/' + channel_id,
             channel_id)
@@ -45,7 +49,7 @@ class PicartoIE(InfoExtractor):
             data=urlencode_postdata({'loadbalancinginfo': channel_id}),
             note='Downloading load balancing info')
 
-        token = self._VALID_URL_RE.match(url).group('token') or 'public'
+        token = mobj.group('token') or 'public'
         params = {
             'con': int(time.time() * 1000),
             'token': token,
@@ -99,9 +103,11 @@ class PicartoIE(InfoExtractor):
 
         return {
             'id': channel_id,
-            'title': self._live_title(channel_id),
+            'title': self._live_title(metadata.get('title') or channel_id),
             'is_live': True,
-            'thumbnail': metadata.get('thumbnails', {}).get('web'),
+            'thumbnail': try_get(metadata, lambda x: x['thumbnails']['web']),
+            'channel': channel_id,
+            'channel_url': 'https://picarto.tv/%s' % channel_id,
             'age_limit': age_limit,
             'formats': formats,
         }

From 9b9b3501c5bee18d608dd2961a80936667f8ece2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Nov 2018 22:55:35 +0700
Subject: [PATCH 168/510] [tnaflixnetwork:embed] Fix extraction (closes #18205)

---
 youtube_dl/extractor/tnaflix.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 0c2f8f119..6798ef4c3 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -18,8 +18,9 @@ from ..utils import (
 class TNAFlixNetworkBaseIE(InfoExtractor):
     # May be overridden in descendants if necessary
     _CONFIG_REGEX = [
-        r'flashvars\.config\s*=\s*escape\("([^"]+)"',
-        r'<input[^>]+name="config\d?" value="([^"]+)"',
+        r'flashvars\.config\s*=\s*escape\("(?P<url>[^"]+)"',
+        r'<input[^>]+name="config\d?" value="(?P<url>[^"]+)"',
+        r'config\s*=\s*(["\'])(?P<url>(?:https?:)?//(?:(?!\1).)+)\1',
     ]
     _HOST = 'tna'
     _VKEY_SUFFIX = ''
@@ -85,7 +86,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         cfg_url = self._proto_relative_url(self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config', default=None), 'http:')
+            self._CONFIG_REGEX, webpage, 'flashvars.config', default=None,
+            group='url'), 'http:')
 
         if not cfg_url:
             inputs = self._hidden_inputs(webpage)

From 2599956c9ff0162e7afbddebb00d73eea6b0403c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Nov 2018 00:07:59 +0700
Subject: [PATCH 169/510] [rte] Add support for new API endpoint (closes
 #18206)

---
 youtube_dl/extractor/rte.py | 129 +++++++++++++++++++++---------------
 1 file changed, 77 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index a6fac6c35..1fbc72915 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -8,7 +8,10 @@ from ..compat import compat_HTTPError
 from ..utils import (
     float_or_none,
     parse_iso8601,
+    str_or_none,
+    try_get,
     unescapeHTML,
+    url_or_none,
     ExtractorError,
 )
 
@@ -17,65 +20,87 @@ class RteBaseIE(InfoExtractor):
     def _real_extract(self, url):
         item_id = self._match_id(url)
 
-        try:
-            json_string = self._download_json(
-                'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id,
-                item_id)
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
-                error_info = self._parse_json(ee.cause.read().decode(), item_id, fatal=False)
-                if error_info:
-                    raise ExtractorError(
-                        '%s said: %s' % (self.IE_NAME, error_info['message']),
-                        expected=True)
-            raise
-
-        # NB the string values in the JSON are stored using XML escaping(!)
-        show = json_string['shows'][0]
-        title = unescapeHTML(show['title'])
-        description = unescapeHTML(show.get('description'))
-        thumbnail = show.get('thumbnail')
-        duration = float_or_none(show.get('duration'), 1000)
-        timestamp = parse_iso8601(show.get('published'))
-
-        mg = show['media:group'][0]
-
+        info_dict = {}
         formats = []
 
-        if mg.get('url'):
-            m = re.match(r'(?P<url>rtmpe?://[^/]+)/(?P<app>.+)/(?P<playpath>mp4:.*)', mg['url'])
-            if m:
-                m = m.groupdict()
-                formats.append({
-                    'url': m['url'] + '/' + m['app'],
-                    'app': m['app'],
-                    'play_path': m['playpath'],
-                    'player_url': url,
-                    'ext': 'flv',
-                    'format_id': 'rtmp',
-                })
+        ENDPOINTS = (
+            'https://feeds.rasset.ie/rteavgen/player/playlist?type=iptv&format=json&showId=',
+            'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=',
+        )
 
-        if mg.get('hls_server') and mg.get('hls_url'):
-            formats.extend(self._extract_m3u8_formats(
-                mg['hls_server'] + mg['hls_url'], item_id, 'mp4',
-                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+        for num, ep_url in enumerate(ENDPOINTS, start=1):
+            try:
+                data = self._download_json(ep_url + item_id, item_id)
+            except ExtractorError as ee:
+                if num < len(ENDPOINTS) or formats:
+                    continue
+                if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+                    error_info = self._parse_json(ee.cause.read().decode(), item_id, fatal=False)
+                    if error_info:
+                        raise ExtractorError(
+                            '%s said: %s' % (self.IE_NAME, error_info['message']),
+                            expected=True)
+                raise
 
-        if mg.get('hds_server') and mg.get('hds_url'):
-            formats.extend(self._extract_f4m_formats(
-                mg['hds_server'] + mg['hds_url'], item_id,
-                f4m_id='hds', fatal=False))
+            # NB the string values in the JSON are stored using XML escaping(!)
+            show = try_get(data, lambda x: x['shows'][0], dict)
+            if not show:
+                continue
+
+            if not info_dict:
+                title = unescapeHTML(show['title'])
+                description = unescapeHTML(show.get('description'))
+                thumbnail = show.get('thumbnail')
+                duration = float_or_none(show.get('duration'), 1000)
+                timestamp = parse_iso8601(show.get('published'))
+                info_dict = {
+                    'id': item_id,
+                    'title': title,
+                    'description': description,
+                    'thumbnail': thumbnail,
+                    'timestamp': timestamp,
+                    'duration': duration,
+                }
+
+            mg = try_get(show, lambda x: x['media:group'][0], dict)
+            if not mg:
+                continue
+
+            if mg.get('url'):
+                m = re.match(r'(?P<url>rtmpe?://[^/]+)/(?P<app>.+)/(?P<playpath>mp4:.*)', mg['url'])
+                if m:
+                    m = m.groupdict()
+                    formats.append({
+                        'url': m['url'] + '/' + m['app'],
+                        'app': m['app'],
+                        'play_path': m['playpath'],
+                        'player_url': url,
+                        'ext': 'flv',
+                        'format_id': 'rtmp',
+                    })
+
+            if mg.get('hls_server') and mg.get('hls_url'):
+                formats.extend(self._extract_m3u8_formats(
+                    mg['hls_server'] + mg['hls_url'], item_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+
+            if mg.get('hds_server') and mg.get('hds_url'):
+                formats.extend(self._extract_f4m_formats(
+                    mg['hds_server'] + mg['hds_url'], item_id,
+                    f4m_id='hds', fatal=False))
+
+            mg_rte_server = str_or_none(mg.get('rte:server'))
+            mg_url = str_or_none(mg.get('url'))
+            if mg_rte_server and mg_url:
+                hds_url = url_or_none(mg_rte_server + mg_url)
+                if hds_url:
+                    formats.extend(self._extract_f4m_formats(
+                        hds_url, item_id, f4m_id='hds', fatal=False))
 
         self._sort_formats(formats)
 
-        return {
-            'id': item_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-        }
+        info_dict['formats'] = formats
+        return info_dict
 
 
 class RteIE(RteBaseIE):

From 0919cd4d011d0545a6afadfab0b71de8c0a4fe02 Mon Sep 17 00:00:00 2001
From: NeroBurner <pyro4hell@gmail.com>
Date: Fri, 16 Nov 2018 18:18:50 +0100
Subject: [PATCH 170/510] [atvat] Fix extraction (closes #18041)

---
 youtube_dl/extractor/atvat.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atvat.py b/youtube_dl/extractor/atvat.py
index 1584d53fc..95e572d70 100644
--- a/youtube_dl/extractor/atvat.py
+++ b/youtube_dl/extractor/atvat.py
@@ -28,8 +28,10 @@ class ATVAtIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         video_data = self._parse_json(unescapeHTML(self._search_regex(
-            r'class="[^"]*jsb_video/FlashPlayer[^"]*"[^>]+data-jsb="([^"]+)"',
-            webpage, 'player data')), display_id)['config']['initial_video']
+            [r'flashPlayerOptions\s*=\s*(["\'])(?P<json>(?:(?!\1).)+)\1',
+             r'class="[^"]*jsb_video/FlashPlayer[^"]*"[^>]+data-jsb="(?P<json>[^"]+)"'],
+            webpage, 'player data', group='json')),
+            display_id)['config']['initial_video']
 
         video_id = video_data['id']
         video_title = video_data['title']

From d0058c76d5e14ffe89e8265fa3d984e28e922d78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Nov 2018 16:59:20 +0700
Subject: [PATCH 171/510] [openload] Use original host during extraction
 (closes #18211)

---
 youtube_dl/extractor/openload.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 2473536fd..cf51e4770 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -243,7 +243,18 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?P<host>
+                            (?:www\.)?
+                            (?:
+                                openload\.(?:co|io|link)|
+                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun)
+                            )
+                        )/
+                        (?:f|embed)/
+                        (?P<id>[a-zA-Z0-9-_]+)
+                    '''
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -334,8 +345,11 @@ class OpenloadIE(InfoExtractor):
             webpage)
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url_pattern = 'https://openload.co/%%s/%s/' % video_id
+        mobj = re.match(self._VALID_URL, url)
+        host = mobj.group('host')
+        video_id = mobj.group('id')
+
+        url_pattern = 'https://%s/%%s/%s/' % (host, video_id)
         headers = {
             'User-Agent': self._USER_AGENT,
         }
@@ -368,7 +382,7 @@ class OpenloadIE(InfoExtractor):
                            r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage,
                           'stream URL'))
 
-        video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
+        video_url = 'https://%s/stream/%s?mime=true' % (host, decoded_id)
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
@@ -379,7 +393,7 @@ class OpenloadIE(InfoExtractor):
         entry = entries[0] if entries else {}
         subtitles = entry.get('subtitles')
 
-        info_dict = {
+        return {
             'id': video_id,
             'title': title,
             'thumbnail': entry.get('thumbnail') or self._og_search_thumbnail(webpage, default=None),
@@ -388,4 +402,3 @@ class OpenloadIE(InfoExtractor):
             'subtitles': subtitles,
             'http_headers': headers,
         }
-        return info_dict

From a640c4d226e7b790fe8db43f1c5bdf2358caf839 Mon Sep 17 00:00:00 2001
From: aviperes <avipr24@gmail.com>
Date: Sat, 17 Nov 2018 15:59:13 +0200
Subject: [PATCH 172/510] [vk] Detect geo restriction

---
 youtube_dl/extractor/vk.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ef8b9bcb7..b52d15ac6 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -293,8 +293,12 @@ class VKIE(VKBaseIE):
             # This video is no longer available, because its author has been blocked.
             'url': 'https://vk.com/video-10639516_456240611',
             'only_matching': True,
-        }
-    ]
+        },
+        {
+            # The video is not available in your region.
+            'url': 'https://vk.com/video-51812607_171445436',
+            'only_matching': True,
+        }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -354,6 +358,9 @@ class VKIE(VKBaseIE):
 
             r'<!>This video is no longer available, because it has been deleted.':
             'Video %s is no longer available, because it has been deleted.',
+
+            r'<!>The video .+? is not available in your region.':
+            'Video %s is not available in your region.',
         }
 
         for error_re, error_msg in ERRORS.items():

From 11d19ff50393cd195af884c6865aff6d89ed66ac Mon Sep 17 00:00:00 2001
From: mttronc <mrtn.mtth@gmx.de>
Date: Thu, 6 Sep 2018 15:41:07 +0200
Subject: [PATCH 173/510] [wwe] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/wwe.py        | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/wwe.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b2b00c86f..87c7d8b0c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1386,6 +1386,7 @@ from .wsj import (
     WSJIE,
     WSJArticleIE,
 )
+from .wwe import WWEIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xfileshare import XFileShareIE
diff --git a/youtube_dl/extractor/wwe.py b/youtube_dl/extractor/wwe.py
new file mode 100644
index 000000000..c471a79f5
--- /dev/null
+++ b/youtube_dl/extractor/wwe.py
@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import urljoin
+
+
+class WWEIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:\w+\.)?wwe.com/(?:.*/)?videos/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.wwe.com/videos/daniel-bryan-vs-andrade-cien-almas-smackdown-live-sept-4-2018',
+        'md5': '30cbc824b51f4010ea885bfcaec76972',
+        'info_dict': {
+            'id': '40048199',
+            'ext': 'mp4',
+            'title': 'Daniel Bryan vs. Andrade "Cien" Almas: SmackDown LIVE, Sept. 4, 2018',
+            'description': 'Still fuming after he and his wife Brie Bella were attacked by The Miz and Maryse last week, Daniel Bryan takes care of some unfinished business with Andrade "Cien" Almas.',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'https://de.wwe.com/videos/gran-metalik-vs-tony-nese-wwe-205-live-sept-4-2018',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        drupal_settings = self._parse_json(
+            self._html_search_regex(
+                r'(?s)Drupal\.settings\s*,\s*({.+?})\);',
+                webpage, 'drupal settings'),
+            display_id)
+
+        player = drupal_settings['WWEVideoLanding']['initialVideo']
+        metadata = player['playlist'][0]
+
+        id = compat_str(metadata['nid'])
+        title = metadata.get('title') or self._og_search_title(webpage)
+        video_url = 'https:' + metadata['file']
+        thumbnail = None
+        if metadata.get('image') is not None:
+            thumbnail = urljoin(url, metadata.get('image'))
+        description = metadata.get('description')
+
+        formats = self._extract_m3u8_formats(video_url, id, 'mp4')
+
+        return {
+            'id': id,
+            'title': title,
+            'formats': formats,
+            'url': video_url,
+            'display_id': display_id,
+            'thumbnail': thumbnail,
+            'description': description,
+        }

From 006374e3aebc3be3f20b7e812c987cdf15b3ae35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Nov 2018 23:59:20 +0700
Subject: [PATCH 174/510] [wwe] Fix issues, extract subtitles and add support
 for playlists (closes #14781, closes #17450)

---
 youtube_dl/extractor/wwe.py | 138 +++++++++++++++++++++++++++++-------
 1 file changed, 111 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/wwe.py b/youtube_dl/extractor/wwe.py
index c471a79f5..bebc77bb5 100644
--- a/youtube_dl/extractor/wwe.py
+++ b/youtube_dl/extractor/wwe.py
@@ -1,20 +1,75 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import urljoin
+from ..utils import (
+    try_get,
+    unescapeHTML,
+    url_or_none,
+    urljoin,
+)
 
 
-class WWEIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?wwe.com/(?:.*/)?videos/(?P<id>[\w-]+)'
+class WWEBaseIE(InfoExtractor):
+    _SUBTITLE_LANGS = {
+        'English': 'en',
+        'Deutsch': 'de',
+    }
+
+    def _extract_entry(self, data, url, video_id=None):
+        video_id = compat_str(video_id or data['nid'])
+        title = data['title']
+
+        formats = self._extract_m3u8_formats(
+            data['file'], video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
+        description = data.get('description')
+        thumbnail = urljoin(url, data.get('image'))
+        series = data.get('show_name')
+        episode = data.get('episode_name')
+
+        subtitles = {}
+        tracks = data.get('tracks')
+        if isinstance(tracks, list):
+            for track in tracks:
+                if not isinstance(track, dict):
+                    continue
+                if track.get('kind') != 'captions':
+                    continue
+                track_file = url_or_none(track.get('file'))
+                if not track_file:
+                    continue
+                label = track.get('label')
+                lang = self._SUBTITLE_LANGS.get(label, label) or 'en'
+                subtitles.setdefault(lang, []).append({
+                    'url': track_file,
+                })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'series': series,
+            'episode': episode,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class WWEIE(WWEBaseIE):
+    _VALID_URL = r'https?://(?:[^/]+\.)?wwe\.com/(?:[^/]+/)*videos/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.wwe.com/videos/daniel-bryan-vs-andrade-cien-almas-smackdown-live-sept-4-2018',
-        'md5': '30cbc824b51f4010ea885bfcaec76972',
+        'md5': '92811c6a14bfc206f7a6a9c5d9140184',
         'info_dict': {
             'id': '40048199',
             'ext': 'mp4',
             'title': 'Daniel Bryan vs. Andrade "Cien" Almas: SmackDown LIVE, Sept. 4, 2018',
-            'description': 'Still fuming after he and his wife Brie Bella were attacked by The Miz and Maryse last week, Daniel Bryan takes care of some unfinished business with Andrade "Cien" Almas.',
+            'description': 'md5:2d7424dbc6755c61a0e649d2a8677f67',
             'thumbnail': r're:^https?://.*\.jpg$',
         }
     }, {
@@ -26,31 +81,60 @@ class WWEIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        drupal_settings = self._parse_json(
+        landing = self._parse_json(
             self._html_search_regex(
-                r'(?s)Drupal\.settings\s*,\s*({.+?})\);',
+                r'(?s)Drupal\.settings\s*,\s*({.+?})\s*\)\s*;',
                 webpage, 'drupal settings'),
-            display_id)
+            display_id)['WWEVideoLanding']
 
-        player = drupal_settings['WWEVideoLanding']['initialVideo']
-        metadata = player['playlist'][0]
+        data = landing['initialVideo']['playlist'][0]
+        video_id = landing.get('initialVideoId')
 
-        id = compat_str(metadata['nid'])
-        title = metadata.get('title') or self._og_search_title(webpage)
-        video_url = 'https:' + metadata['file']
-        thumbnail = None
-        if metadata.get('image') is not None:
-            thumbnail = urljoin(url, metadata.get('image'))
-        description = metadata.get('description')
+        info = self._extract_entry(data, url, video_id)
+        info['display_id'] = display_id
+        return info
 
-        formats = self._extract_m3u8_formats(video_url, id, 'mp4')
 
-        return {
-            'id': id,
-            'title': title,
-            'formats': formats,
-            'url': video_url,
-            'display_id': display_id,
-            'thumbnail': thumbnail,
-            'description': description,
-        }
+class WWEPlaylistIE(WWEBaseIE):
+    _VALID_URL = r'https?://(?:[^/]+\.)?wwe\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.wwe.com/shows/raw/2018-11-12',
+        'info_dict': {
+            'id': '2018-11-12',
+        },
+        'playlist_mincount': 11,
+    }, {
+        'url': 'http://www.wwe.com/article/walk-the-prank-wwe-edition',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.wwe.com/shows/wwenxt/article/matt-riddle-interview',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if WWEIE.suitable(url) else super(WWEPlaylistIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        entries = []
+        for mobj in re.finditer(
+                r'data-video\s*=\s*(["\'])(?P<data>{.+?})\1', webpage):
+            video = self._parse_json(
+                mobj.group('data'), display_id, transform_source=unescapeHTML,
+                fatal=False)
+            if not video:
+                continue
+            data = try_get(video, lambda x: x['playlist'][0], dict)
+            if not data:
+                continue
+            try:
+                entry = self._extract_entry(data, url)
+            except Exception:
+                continue
+            entry['extractor_key'] = WWEIE.ie_key()
+            entries.append(entry)
+
+        return self.playlist_result(entries, display_id)

From 02df855e1339942c00f365dc59a0e418abb4b26f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Nov 2018 00:07:40 +0700
Subject: [PATCH 175/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index fa5de8b04..15daa1bec 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Extractors
++ [wwe] Extract subtitles
++ [wwe] Add support for playlistst (#14781)
++ [wwe] Add support for wwe.com (#14781, #17450)
+* [vk] Detect geo restriction (#17767)
+* [openload] Use original host during extraction (#18211)
+* [atvat] Fix extraction (#18041)
++ [rte] Add support for new API endpoint (#18206)
+* [tnaflixnetwork:embed] Fix extraction (#18205)
+* [picarto] Use API and add token support (#16518)
++ [zype] Add support for player.zype.com (#18143)
+* [vivo] Fix extraction (#18139)
+* [ruutu] Update API endpoint (#18138)
+
+
 version 2018.11.07
 
 Extractors

From 5bb04792696c41f66f2114b0cc05b01135fa1f68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Nov 2018 00:11:54 +0700
Subject: [PATCH 176/510] release 2018.11.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7607e0e03..905576364 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.11.07
+[debug] youtube-dl version 2018.11.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 15daa1bec..0083c4631 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.11.18
 
 Extractors
 + [wwe] Extract subtitles
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 24c3254c3..9009f7e9e 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1080,6 +1080,7 @@
  - **wrzuta.pl:playlist**
  - **WSJ**: Wall Street Journal
  - **WSJArticle**
+ - **WWE**
  - **XBef**
  - **XboxClips**
  - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV, FastVideo.me
@@ -1139,3 +1140,4 @@
  - **ZDF**
  - **ZDFChannel**
  - **zingmp3**: mp3.zing.vn
+ - **Zype**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7f32ad36c..7f5ad7bf4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.11.07'
+__version__ = '2018.11.18'

From 4167148fa45e43a93ed202e5923223b7797340ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Nov 2018 01:07:54 +0700
Subject: [PATCH 177/510] [nova:embed] Fix extraction (closes #18222)

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 80186ec50..901f44b54 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -35,7 +35,7 @@ class NovaEmbedIE(InfoExtractor):
 
         bitrates = self._parse_json(
             self._search_regex(
-                r'(?s)bitrates\s*=\s*({.+?})\s*;', webpage, 'formats'),
+                r'(?s)(?:src|bitrates)\s*=\s*({.+?})\s*;', webpage, 'formats'),
             video_id, transform_source=js_to_json)
 
         QUALITIES = ('lq', 'mq', 'hq', 'hd')

From 1febf99da1924c46a491790639371f4ee9069193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Nov 2018 06:26:08 +0700
Subject: [PATCH 178/510] [pornhub] Add pornhub.net alias

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 19eaf389f..7ee64dbf6 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -27,7 +27,7 @@ class PornHubIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:[^/]+\.)?pornhub\.com/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
+                            (?:[^/]+\.)?pornhub\.(?:com|net)/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
                             (?:www\.)?thumbzilla\.com/video/
                         )
                         (?P<id>[\da-z]+)
@@ -340,7 +340,7 @@ class PornHubPlaylistBaseIE(InfoExtractor):
 
 
 class PornHubPlaylistIE(PornHubPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.com/playlist/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.(?:com|net)/playlist/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/playlist/4667351',
         'info_dict': {
@@ -355,7 +355,7 @@ class PornHubPlaylistIE(PornHubPlaylistBaseIE):
 
 
 class PornHubUserVideosIE(PornHubPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.com/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos'
+    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.(?:com|net)/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos'
     _TESTS = [{
         'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
         'info_dict': {
@@ -393,6 +393,9 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
     }, {
         'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
         'only_matching': True,
+    }, {
+        'url': 'https://www.pornhub.net/view_video.php?viewkey=203640933',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f97c099131f625104f64a99a02a8c9894620171a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Nov 2018 11:14:46 +0700
Subject: [PATCH 179/510] [pornhub] Move test to correct place

---
 youtube_dl/extractor/pornhub.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 7ee64dbf6..c9c884095 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -121,6 +121,9 @@ class PornHubIE(InfoExtractor):
     }, {
         'url': 'http://www.pornhub.com/video/show?viewkey=648719015',
         'only_matching': True,
+    }, {
+        'url': 'https://www.pornhub.net/view_video.php?viewkey=203640933',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -393,9 +396,6 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
     }, {
         'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
         'only_matching': True,
-    }, {
-        'url': 'https://www.pornhub.net/view_video.php?viewkey=203640933',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 964b989dc88c37b027481fb01de835b1e796ba5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Nov 2018 20:44:51 +0700
Subject: [PATCH 180/510] [americastestkitchen] Add support for zype embeds
 (closes #18225)

---
 youtube_dl/extractor/americastestkitchen.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index 01736872d..8b32aa886 100644
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -43,10 +43,6 @@ class AmericasTestKitchenIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        partner_id = self._search_regex(
-            r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
-            webpage, 'kaltura partner id')
-
         video_data = self._parse_json(
             self._search_regex(
                 r'window\.__INITIAL_STATE__\s*=\s*({.+?})\s*;\s*</script>',
@@ -58,7 +54,18 @@ class AmericasTestKitchenIE(InfoExtractor):
             (lambda x: x['episodeDetail']['content']['data'],
              lambda x: x['videoDetail']['content']['data']), dict)
         ep_meta = ep_data.get('full_video', {})
-        external_id = ep_data.get('external_id') or ep_meta['external_id']
+
+        zype_id = ep_meta.get('zype_id')
+        if zype_id:
+            embed_url = 'https://player.zype.com/embed/%s.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ' % zype_id
+            ie_key = 'Zype'
+        else:
+            partner_id = self._search_regex(
+                r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
+                webpage, 'kaltura partner id')
+            external_id = ep_data.get('external_id') or ep_meta['external_id']
+            embed_url = 'kaltura:%s:%s' % (partner_id, external_id)
+            ie_key = 'Kaltura'
 
         title = ep_data.get('title') or ep_meta.get('title')
         description = clean_html(ep_meta.get('episode_description') or ep_data.get(
@@ -72,8 +79,8 @@ class AmericasTestKitchenIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'url': 'kaltura:%s:%s' % (partner_id, external_id),
-            'ie_key': 'Kaltura',
+            'url': embed_url,
+            'ie_key': ie_key,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 9b27a78a881bceb9d62f3364399d7572e8e2be24 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 18 Nov 2018 16:13:46 +0100
Subject: [PATCH 181/510] [kaltura] limit requested MediaEntry fields

---
 youtube_dl/extractor/kaltura.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 04f68fce4..fdf7f5bbc 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -192,6 +192,8 @@ class KalturaIE(InfoExtractor):
                 'entryId': video_id,
                 'service': 'baseentry',
                 'ks': '{1:result:ks}',
+                'responseProfile:fields': 'createdAt,dataUrl,duration,name,plays,thumbnailUrl,userId',
+                'responseProfile:type': 1,
             },
             {
                 'action': 'getbyentryid',

From 8578ea4dcb17834ee3843e0e337c15af706f9803 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 18 Nov 2018 16:15:10 +0100
Subject: [PATCH 182/510] [bitchute] use _html_search_regex for title
 extraction

---
 youtube_dl/extractor/bitchute.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 446a1ab19..43b4732aa 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -37,7 +37,7 @@ class BitChuteIE(InfoExtractor):
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
             })
 
-        title = self._search_regex(
+        title = self._html_search_regex(
             (r'<[^>]+\bid=["\']video-title[^>]+>([^<]+)', r'<title>([^<]+)'),
             webpage, 'title', default=None) or self._html_search_meta(
             'description', webpage, 'title',

From 2e1280ed432257244ea52a47efe6a7f0e226b897 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 19 Nov 2018 18:15:51 +0100
Subject: [PATCH 183/510] [sixplay] fix format extraction

---
 youtube_dl/extractor/sixplay.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index 207ab4477..0c4f865ef 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -64,7 +64,7 @@ class SixPlayIE(InfoExtractor):
         for asset in clip_data['assets']:
             asset_url = asset.get('full_physical_path')
             protocol = asset.get('protocol')
-            if not asset_url or protocol == 'primetime' or asset_url in urls:
+            if not asset_url or protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264' or asset_url in urls:
                 continue
             urls.append(asset_url)
             container = asset.get('video_container')
@@ -81,19 +81,17 @@ class SixPlayIE(InfoExtractor):
                         if not urlh:
                             continue
                         asset_url = urlh.geturl()
-                    asset_url = re.sub(r'/([^/]+)\.ism/[^/]*\.m3u8', r'/\1.ism/\1.m3u8', asset_url)
-                    formats.extend(self._extract_m3u8_formats(
-                        asset_url, video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
-                    formats.extend(self._extract_f4m_formats(
-                        asset_url.replace('.m3u8', '.f4m'),
-                        video_id, f4m_id='hds', fatal=False))
-                    formats.extend(self._extract_mpd_formats(
-                        asset_url.replace('.m3u8', '.mpd'),
-                        video_id, mpd_id='dash', fatal=False))
-                    formats.extend(self._extract_ism_formats(
-                        re.sub(r'/[^/]+\.m3u8', '/Manifest', asset_url),
-                        video_id, ism_id='mss', fatal=False))
+                    for i in range(3, 0, -1):
+                        asset_url = asset_url = asset_url.replace('_sd1/', '_sd%d/' % i)
+                        m3u8_formats = self._extract_m3u8_formats(
+                            asset_url, video_id, 'mp4', 'm3u8_native',
+                            m3u8_id='hls', fatal=False)
+                        formats.extend(m3u8_formats)
+                        formats.extend(self._extract_mpd_formats(
+                            asset_url.replace('.m3u8', '.mpd'),
+                            video_id, mpd_id='dash', fatal=False))
+                        if m3u8_formats:
+                            break
                 else:
                     formats.extend(self._extract_m3u8_formats(
                         asset_url, video_id, 'mp4', 'm3u8_native',

From 15ed5a27840e748d9f786c50b78a4c6326e9f186 Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Tue, 20 Nov 2018 20:50:40 +0100
Subject: [PATCH 184/510] [nzz] Relax kaltura regex

---
 youtube_dl/extractor/nzz.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nzz.py b/youtube_dl/extractor/nzz.py
index 2d352f53f..61ee77adb 100644
--- a/youtube_dl/extractor/nzz.py
+++ b/youtube_dl/extractor/nzz.py
@@ -11,20 +11,27 @@ from ..utils import (
 
 class NZZIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?nzz\.ch/(?:[^/]+/)*[^/?#]+-ld\.(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nzz.ch/zuerich/gymizyte/gymizyte-schreiben-schueler-heute-noch-diktate-ld.9153',
         'info_dict': {
             'id': '9153',
         },
         'playlist_mincount': 6,
-    }
+    }, {
+        'url': 'https://www.nzz.ch/video/nzz-standpunkte/cvp-auf-der-suche-nach-dem-mass-der-mitte-ld.1368112',
+        'info_dict': {
+            'id': '1368112',
+        },
+        'playlist_count': 1,
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
 
         entries = []
-        for player_element in re.findall(r'(<[^>]+class="kalturaPlayer"[^>]*>)', webpage):
+        for player_element in re.findall(
+                r'(<[^>]+class="kalturaPlayer[^"]*"[^>]*>)', webpage):
             player_params = extract_attributes(player_element)
             if player_params.get('data-type') not in ('kaltura_singleArticle',):
                 self.report_warning('Unsupported player type')

From 05bd5e9c77e0e8acb95f47396be4c970fc9f39c4 Mon Sep 17 00:00:00 2001
From: Austin de Coup-Crank <austindcc@gmail.com>
Date: Fri, 26 Oct 2018 19:15:44 -0700
Subject: [PATCH 185/510] [ciscolive] Add extractor

---
 youtube_dl/extractor/ciscolive.py  | 136 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 137 insertions(+)
 create mode 100644 youtube_dl/extractor/ciscolive.py

diff --git a/youtube_dl/extractor/ciscolive.py b/youtube_dl/extractor/ciscolive.py
new file mode 100644
index 000000000..2db7aad2c
--- /dev/null
+++ b/youtube_dl/extractor/ciscolive.py
@@ -0,0 +1,136 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlparse,
+    compat_parse_qs
+)
+from ..utils import (
+    clean_html,
+    int_or_none,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class CiscoLiveIE(InfoExtractor):
+    IE_NAME = 'ciscolive'
+    _VALID_URL = r'(?:https?://)?ciscolive\.cisco\.com/on-demand-library/\??(?P<query>[^#]+)#/(?:session/(?P<id>.+))?$'
+    _TESTS = [
+        {
+            'url': 'https://ciscolive.cisco.com/on-demand-library/?#/session/1423353499155001FoSs',
+            'md5': 'c98acf395ed9c9f766941c70f5352e22',
+            'info_dict': {
+                'id': '5803694304001',
+                'ext': 'mp4',
+                'title': '13 Smart Automations to Monitor Your Cisco IOS Network',
+                'description': 'md5:ec4a436019e09a918dec17714803f7cc',
+                'timestamp': 1530305395,
+                'uploader_id': '5647924234001',
+                'upload_date': '20180629',
+                'location': '16B Mezz.',
+            },
+        },
+        {
+            'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
+            'md5': '993d4cf051f6174059328b1dce8e94bd',
+            'info_dict': {
+                'upload_date': '20180629',
+                'title': 'DevNet Panel-Applying Design Thinking to Building Products in Cisco',
+                'timestamp': 1530316421,
+                'uploader_id': '5647924234001',
+                'id': '5803751616001',
+                'description': 'md5:5f144575cd6848117fe2f756855b038b',
+                'location': 'WoS, DevNet Theater',
+                'ext': 'mp4',
+            },
+        },
+        {
+            'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
+            'md5': '80e0c3b87e373fe3a3316b934b8915bf',
+            'info_dict': {
+                'upload_date': '20180629',
+                'title': 'Beating the CCIE Routing & Switching',
+                'timestamp': 1530311842,
+                'uploader_id': '5647924234001',
+                'id': '5803735679001',
+                'description': 'md5:e71970799e92d7f5ff57ae23f64b0929',
+                'location': 'Tulúm 02',
+                'ext': 'mp4',
+            },
+        }
+    ]
+
+    # These appear to be constant across all Cisco Live presentations
+    # and are not tied to any user session or event
+    RAINFOCUS_API_URL = 'https://events.rainfocus.com/api/%s'
+    RAINFOCUS_APIPROFILEID = 'Na3vqYdAlJFSxhYTYQGuMbpafMqftalz'
+    RAINFOCUS_WIDGETID = 'n6l4Lo05R8fiy3RpUBm447dZN8uNWoye'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5647924234001/SyK2FdqjM_default/index.html?videoId=%s'
+
+    def _parse_rf_item(self, rf_item):
+        ''' Parses metadata and passes to Brightcove extractor '''
+        event_name = rf_item.get('eventName')
+        title = rf_item['title']
+        description = clean_html(rf_item.get('abstract'))
+        presenter_name = try_get(rf_item, lambda x: x['participants'][0]['fullName'])
+        bc_id = rf_item['videos'][0]['url']
+        bc_url = self.BRIGHTCOVE_URL_TEMPLATE % bc_id
+        duration = int_or_none(try_get(rf_item, lambda x: x['times'][0]['length']))
+        location = try_get(rf_item, lambda x: x['times'][0]['room'])
+
+        if duration:
+            duration = duration * 60
+
+        return {
+            '_type': 'url_transparent',
+            'creator': presenter_name,
+            'description': description,
+            'duration': duration,
+            'ie_key': 'BrightcoveNew',
+            'location': location,
+            'series': event_name,
+            'title': title,
+            'url': bc_url,
+        }
+
+    def _check_bc_id_exists(self, rf_item):
+        ''' Checks for the existence of a Brightcove URL in an API result '''
+        bc_id = try_get(rf_item, lambda x: x['videos'][0]['url'])
+        if bc_id:
+            if bc_id.strip().isdigit():
+                return rf_item
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        HEADERS = {
+            'Origin': 'https://ciscolive.cisco.com',
+            'rfApiProfileId': self.RAINFOCUS_APIPROFILEID,
+            'rfWidgetId': self.RAINFOCUS_WIDGETID,
+            'Referer': url,
+        }
+        # Single session URL (single video)
+        if mobj.group('id'):
+            rf_id = mobj.group('id')
+            request = self.RAINFOCUS_API_URL % 'session'
+            data = urlencode_postdata({'id': rf_id})
+            rf_result = self._download_json(request, rf_id, data=data, headers=HEADERS)
+            rf_item = self._check_bc_id_exists(rf_result['items'][0])
+            return self._parse_rf_item(rf_item)
+        else:
+            # Filter query URL (multiple videos)
+            rf_query = compat_parse_qs((compat_urllib_parse_urlparse(url).query))
+            rf_query['type'] = 'session'
+            rf_query['size'] = 1000
+            data = urlencode_postdata(rf_query)
+            request = self.RAINFOCUS_API_URL % 'search'
+            rf_results = self._download_json(request, 'Filter query', data=data, headers=HEADERS)
+            entries = [
+                self._parse_rf_item(rf_item)
+                for rf_item
+                in rf_results['sectionList'][0]['items']
+                if self._check_bc_id_exists(rf_item)
+            ]
+            return self.playlist_result(entries, 'Filter query')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 87c7d8b0c..2c5988a14 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -194,6 +194,7 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
+from .ciscolive import CiscoLiveIE
 from .cjsw import CJSWIE
 from .cliphunter import CliphunterIE
 from .clippit import ClippitIE

From 6a6d7f064178427d28986884524bd3434f0ca957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Nov 2018 05:25:43 +0700
Subject: [PATCH 186/510] [ciscolive] Fix issues and improve extraction (closes
 #17984)

---
 youtube_dl/extractor/ciscolive.py  | 176 ++++++++++++++---------------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 87 insertions(+), 94 deletions(-)

diff --git a/youtube_dl/extractor/ciscolive.py b/youtube_dl/extractor/ciscolive.py
index 2db7aad2c..32f645713 100644
--- a/youtube_dl/extractor/ciscolive.py
+++ b/youtube_dl/extractor/ciscolive.py
@@ -1,84 +1,49 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urllib_parse_urlparse,
-    compat_parse_qs
 )
 from ..utils import (
     clean_html,
+    float_or_none,
     int_or_none,
     try_get,
     urlencode_postdata,
 )
 
 
-class CiscoLiveIE(InfoExtractor):
-    IE_NAME = 'ciscolive'
-    _VALID_URL = r'(?:https?://)?ciscolive\.cisco\.com/on-demand-library/\??(?P<query>[^#]+)#/(?:session/(?P<id>.+))?$'
-    _TESTS = [
-        {
-            'url': 'https://ciscolive.cisco.com/on-demand-library/?#/session/1423353499155001FoSs',
-            'md5': 'c98acf395ed9c9f766941c70f5352e22',
-            'info_dict': {
-                'id': '5803694304001',
-                'ext': 'mp4',
-                'title': '13 Smart Automations to Monitor Your Cisco IOS Network',
-                'description': 'md5:ec4a436019e09a918dec17714803f7cc',
-                'timestamp': 1530305395,
-                'uploader_id': '5647924234001',
-                'upload_date': '20180629',
-                'location': '16B Mezz.',
-            },
-        },
-        {
-            'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
-            'md5': '993d4cf051f6174059328b1dce8e94bd',
-            'info_dict': {
-                'upload_date': '20180629',
-                'title': 'DevNet Panel-Applying Design Thinking to Building Products in Cisco',
-                'timestamp': 1530316421,
-                'uploader_id': '5647924234001',
-                'id': '5803751616001',
-                'description': 'md5:5f144575cd6848117fe2f756855b038b',
-                'location': 'WoS, DevNet Theater',
-                'ext': 'mp4',
-            },
-        },
-        {
-            'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
-            'md5': '80e0c3b87e373fe3a3316b934b8915bf',
-            'info_dict': {
-                'upload_date': '20180629',
-                'title': 'Beating the CCIE Routing & Switching',
-                'timestamp': 1530311842,
-                'uploader_id': '5647924234001',
-                'id': '5803735679001',
-                'description': 'md5:e71970799e92d7f5ff57ae23f64b0929',
-                'location': 'Tulúm 02',
-                'ext': 'mp4',
-            },
-        }
-    ]
-
+class CiscoLiveBaseIE(InfoExtractor):
     # These appear to be constant across all Cisco Live presentations
     # and are not tied to any user session or event
     RAINFOCUS_API_URL = 'https://events.rainfocus.com/api/%s'
-    RAINFOCUS_APIPROFILEID = 'Na3vqYdAlJFSxhYTYQGuMbpafMqftalz'
-    RAINFOCUS_WIDGETID = 'n6l4Lo05R8fiy3RpUBm447dZN8uNWoye'
+    RAINFOCUS_API_PROFILE_ID = 'Na3vqYdAlJFSxhYTYQGuMbpafMqftalz'
+    RAINFOCUS_WIDGET_ID = 'n6l4Lo05R8fiy3RpUBm447dZN8uNWoye'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5647924234001/SyK2FdqjM_default/index.html?videoId=%s'
 
+    HEADERS = {
+        'Origin': 'https://ciscolive.cisco.com',
+        'rfApiProfileId': RAINFOCUS_API_PROFILE_ID,
+        'rfWidgetId': RAINFOCUS_WIDGET_ID,
+    }
+
+    def _call_api(self, ep, rf_id, query, referrer):
+        headers = self.HEADERS.copy()
+        headers['Referer'] = referrer
+        return self._download_json(
+            self.RAINFOCUS_API_URL % ep, rf_id, data=urlencode_postdata(query),
+            headers=headers)
+
     def _parse_rf_item(self, rf_item):
-        ''' Parses metadata and passes to Brightcove extractor '''
         event_name = rf_item.get('eventName')
         title = rf_item['title']
         description = clean_html(rf_item.get('abstract'))
         presenter_name = try_get(rf_item, lambda x: x['participants'][0]['fullName'])
         bc_id = rf_item['videos'][0]['url']
         bc_url = self.BRIGHTCOVE_URL_TEMPLATE % bc_id
-        duration = int_or_none(try_get(rf_item, lambda x: x['times'][0]['length']))
+        duration = float_or_none(try_get(rf_item, lambda x: x['times'][0]['length']))
         location = try_get(rf_item, lambda x: x['times'][0]['room'])
 
         if duration:
@@ -86,51 +51,76 @@ class CiscoLiveIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'creator': presenter_name,
+            'url': bc_url,
+            'ie_key': 'BrightcoveNew',
+            'title': title,
             'description': description,
             'duration': duration,
-            'ie_key': 'BrightcoveNew',
+            'creator': presenter_name,
             'location': location,
             'series': event_name,
-            'title': title,
-            'url': bc_url,
         }
 
-    def _check_bc_id_exists(self, rf_item):
-        ''' Checks for the existence of a Brightcove URL in an API result '''
-        bc_id = try_get(rf_item, lambda x: x['videos'][0]['url'])
-        if bc_id:
-            if bc_id.strip().isdigit():
-                return rf_item
+
+class CiscoLiveSessionIE(CiscoLiveBaseIE):
+    _VALID_URL = r'https?://ciscolive\.cisco\.com/on-demand-library/\??[^#]*#/session/(?P<id>[^/?&]+)'
+    _TEST = {
+        'url': 'https://ciscolive.cisco.com/on-demand-library/?#/session/1423353499155001FoSs',
+        'md5': 'c98acf395ed9c9f766941c70f5352e22',
+        'info_dict': {
+            'id': '5803694304001',
+            'ext': 'mp4',
+            'title': '13 Smart Automations to Monitor Your Cisco IOS Network',
+            'description': 'md5:ec4a436019e09a918dec17714803f7cc',
+            'timestamp': 1530305395,
+            'upload_date': '20180629',
+            'uploader_id': '5647924234001',
+            'location': '16B Mezz.',
+        },
+        'params': {
+            'proxy': '127.0.0.1:8118',
+        }
+    }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        HEADERS = {
-            'Origin': 'https://ciscolive.cisco.com',
-            'rfApiProfileId': self.RAINFOCUS_APIPROFILEID,
-            'rfWidgetId': self.RAINFOCUS_WIDGETID,
-            'Referer': url,
+        rf_id = self._match_id(url)
+        rf_result = self._call_api('session', rf_id, {'id': rf_id}, url)
+        return self._parse_rf_item(rf_result['items'][0])
+
+
+class CiscoLiveSearchIE(CiscoLiveBaseIE):
+    _VALID_URL = r'https?://ciscolive\.cisco\.com/on-demand-library/'
+    _TESTS = [{
+        'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
+        'info_dict': {
+            'title': 'Filter query',
+        },
+        'playlist_count': 5,
+        'params': {
+            'proxy': '127.0.0.1:8118',
         }
-        # Single session URL (single video)
-        if mobj.group('id'):
-            rf_id = mobj.group('id')
-            request = self.RAINFOCUS_API_URL % 'session'
-            data = urlencode_postdata({'id': rf_id})
-            rf_result = self._download_json(request, rf_id, data=data, headers=HEADERS)
-            rf_item = self._check_bc_id_exists(rf_result['items'][0])
-            return self._parse_rf_item(rf_item)
-        else:
-            # Filter query URL (multiple videos)
-            rf_query = compat_parse_qs((compat_urllib_parse_urlparse(url).query))
-            rf_query['type'] = 'session'
-            rf_query['size'] = 1000
-            data = urlencode_postdata(rf_query)
-            request = self.RAINFOCUS_API_URL % 'search'
-            rf_results = self._download_json(request, 'Filter query', data=data, headers=HEADERS)
-            entries = [
-                self._parse_rf_item(rf_item)
-                for rf_item
-                in rf_results['sectionList'][0]['items']
-                if self._check_bc_id_exists(rf_item)
-            ]
-            return self.playlist_result(entries, 'Filter query')
+    }, {
+        'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if CiscoLiveSessionIE.suitable(url) else super(CiscoLiveSearchIE, cls).suitable(url)
+
+    @staticmethod
+    def _check_bc_id_exists(rf_item):
+        return int_or_none(try_get(rf_item, lambda x: x['videos'][0]['url'])) is not None
+
+    def _real_extract(self, url):
+        rf_query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        rf_query['type'] = 'session'
+        rf_query['size'] = 1000
+        rf_results = self._call_api('search', None, rf_query, url)
+        entries = [
+            self._parse_rf_item(rf_item)
+            for rf_item
+            in rf_results['sectionList'][0]['items']
+            if self._check_bc_id_exists(rf_item)
+        ]
+        return self.playlist_result(entries, playlist_title='Filter query')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2c5988a14..60e6175b1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -194,7 +194,10 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
-from .ciscolive import CiscoLiveIE
+from .ciscolive import (
+    CiscoLiveSessionIE,
+    CiscoLiveSearchIE,
+)
 from .cjsw import CJSWIE
 from .cliphunter import CliphunterIE
 from .clippit import ClippitIE

From 183417a50fd68c0c63b1d0621c6a0b44fbf2ac52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Nov 2018 06:04:34 +0700
Subject: [PATCH 187/510] [ciscolive:search] Add support for pagination

---
 youtube_dl/extractor/ciscolive.py | 58 ++++++++++++++++++++-----------
 1 file changed, 37 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/ciscolive.py b/youtube_dl/extractor/ciscolive.py
index 32f645713..c99b6ee58 100644
--- a/youtube_dl/extractor/ciscolive.py
+++ b/youtube_dl/extractor/ciscolive.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
+
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
@@ -29,12 +31,12 @@ class CiscoLiveBaseIE(InfoExtractor):
         'rfWidgetId': RAINFOCUS_WIDGET_ID,
     }
 
-    def _call_api(self, ep, rf_id, query, referrer):
+    def _call_api(self, ep, rf_id, query, referrer, note=None):
         headers = self.HEADERS.copy()
         headers['Referer'] = referrer
         return self._download_json(
-            self.RAINFOCUS_API_URL % ep, rf_id, data=urlencode_postdata(query),
-            headers=headers)
+            self.RAINFOCUS_API_URL % ep, rf_id, note=note,
+            data=urlencode_postdata(query), headers=headers)
 
     def _parse_rf_item(self, rf_item):
         event_name = rf_item.get('eventName')
@@ -77,9 +79,6 @@ class CiscoLiveSessionIE(CiscoLiveBaseIE):
             'uploader_id': '5647924234001',
             'location': '16B Mezz.',
         },
-        'params': {
-            'proxy': '127.0.0.1:8118',
-        }
     }
 
     def _real_extract(self, url):
@@ -93,12 +92,9 @@ class CiscoLiveSearchIE(CiscoLiveBaseIE):
     _TESTS = [{
         'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
         'info_dict': {
-            'title': 'Filter query',
+            'title': 'Search query',
         },
         'playlist_count': 5,
-        'params': {
-            'proxy': '127.0.0.1:8118',
-        }
     }, {
         'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
         'only_matching': True,
@@ -112,15 +108,35 @@ class CiscoLiveSearchIE(CiscoLiveBaseIE):
     def _check_bc_id_exists(rf_item):
         return int_or_none(try_get(rf_item, lambda x: x['videos'][0]['url'])) is not None
 
+    def _entries(self, query, url):
+        query['size'] = 50
+        query['from'] = 0
+        for page_num in itertools.count(1):
+            results = self._call_api(
+                'search', None, query, url,
+                'Downloading search JSON page %d' % page_num)
+            sl = try_get(results, lambda x: x['sectionList'][0], dict)
+            if sl:
+                results = sl
+            items = results.get('items')
+            if not items or not isinstance(items, list):
+                break
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                if not self._check_bc_id_exists(item):
+                    continue
+                yield self._parse_rf_item(item)
+            size = int_or_none(results.get('size'))
+            if size is not None:
+                query['size'] = size
+            total = int_or_none(results.get('total'))
+            if total is not None and query['from'] + query['size'] > total:
+                break
+            query['from'] += query['size']
+
     def _real_extract(self, url):
-        rf_query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
-        rf_query['type'] = 'session'
-        rf_query['size'] = 1000
-        rf_results = self._call_api('search', None, rf_query, url)
-        entries = [
-            self._parse_rf_item(rf_item)
-            for rf_item
-            in rf_results['sectionList'][0]['items']
-            if self._check_bc_id_exists(rf_item)
-        ]
-        return self.playlist_result(entries, playlist_title='Filter query')
+        query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        query['type'] = 'session'
+        return self.playlist_result(
+            self._entries(query, url), playlist_title='Search query')

From 6c882aa8991383e1c39a6457cbde5dcab260bff5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 09:44:10 +0100
Subject: [PATCH 188/510] [loc] relax _VALID_URL regex and improve formats
 extraction

---
 youtube_dl/extractor/libraryofcongress.py | 37 +++++++++++++++--------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
index 40295a30b..1e5c82c66 100644
--- a/youtube_dl/extractor/libraryofcongress.py
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -16,7 +16,7 @@ from ..utils import (
 class LibraryOfCongressIE(InfoExtractor):
     IE_NAME = 'loc'
     IE_DESC = 'Library of Congress'
-    _VALID_URL = r'https?://(?:www\.)?loc\.gov/(?:item/|today/cyberlc/feature_wdesc\.php\?.*\brec=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?loc\.gov/(?:item/|today/cyberlc/feature_wdesc\.php\?.*\brec=)(?P<id>[0-9a-z_.]+)'
     _TESTS = [{
         # embedded via <div class="media-player"
         'url': 'http://loc.gov/item/90716351/',
@@ -57,6 +57,12 @@ class LibraryOfCongressIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.loc.gov/item/ihas.200197114/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.loc.gov/item/afc1981005_afs20503/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -67,12 +73,13 @@ class LibraryOfCongressIE(InfoExtractor):
             (r'id=(["\'])media-player-(?P<id>.+?)\1',
              r'<video[^>]+id=(["\'])uuid-(?P<id>.+?)\1',
              r'<video[^>]+data-uuid=(["\'])(?P<id>.+?)\1',
-             r'mediaObjectId\s*:\s*(["\'])(?P<id>.+?)\1'),
+             r'mediaObjectId\s*:\s*(["\'])(?P<id>.+?)\1',
+             r'data-tab="share-media-(?P<id>[0-9A-F]{32})"'),
             webpage, 'media id', group='id')
 
         data = self._download_json(
             'https://media.loc.gov/services/v1/media?id=%s&context=json' % media_id,
-            video_id)['mediaObject']
+            media_id)['mediaObject']
 
         derivative = data['derivatives'][0]
         media_url = derivative['derivativeUrl']
@@ -89,25 +96,29 @@ class LibraryOfCongressIE(InfoExtractor):
         if ext not in ('mp4', 'mp3'):
             media_url += '.mp4' if is_video else '.mp3'
 
-        if 'vod/mp4:' in media_url:
-            formats = [{
-                'url': media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8',
+        formats = []
+        if '/vod/mp4:' in media_url:
+            formats.append({
+                'url': media_url.replace('/vod/mp4:', '/hls-vod/media/') + '.m3u8',
                 'format_id': 'hls',
                 'ext': 'mp4',
                 'protocol': 'm3u8_native',
                 'quality': 1,
-            }]
-        elif 'vod/mp3:' in media_url:
-            formats = [{
-                'url': media_url.replace('vod/mp3:', ''),
-                'vcodec': 'none',
-            }]
+            })
+        http_format = {
+            'url': re.sub(r'(://[^/]+/)(?:[^/]+/)*(?:mp4|mp3):', r'\1', media_url),
+            'format_id': 'http',
+            'quality': 1,
+        }
+        if not is_video:
+            http_format['vcodec'] = 'none'
+        formats.append(http_format)
 
         download_urls = set()
         for m in re.finditer(
                 r'<option[^>]+value=(["\'])(?P<url>.+?)\1[^>]+data-file-download=[^>]+>\s*(?P<id>.+?)(?:(?:&nbsp;|\s+)\((?P<size>.+?)\))?\s*<', webpage):
             format_id = m.group('id').lower()
-            if format_id == 'gif':
+            if format_id in ('gif', 'jpeg'):
                 continue
             download_url = m.group('url')
             if download_url in download_urls:

From 35328915b5fe5c8915b924cfbc54bbdd6d6d1430 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 09:46:13 +0100
Subject: [PATCH 189/510] [foxsports] fix extraction(closes #17543)

---
 youtube_dl/extractor/foxsports.py   | 17 +++--------------
 youtube_dl/extractor/theplatform.py |  5 +++--
 2 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index 985542727..596fded20 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class FoxSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*video/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.foxsports.com/tennessee/video/432609859715',
@@ -28,16 +28,5 @@ class FoxSportsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        config = self._parse_json(
-            self._html_search_regex(
-                r"""class="[^"]*(?:fs-player|platformPlayer-wrapper)[^"]*".+?data-player-config='([^']+)'""",
-                webpage, 'data player config'),
-            video_id)
-
-        return self.url_result(smuggle_url(update_url_query(
-            config['releaseURL'], {
-                'mbr': 'true',
-                'switch': 'http',
-            }), {'force_smil_url': True}))
+        return self.url_result(
+            'https://feed.theplatform.com/f/BKQ29B/foxsports-all?byId=' + video_id, 'ThePlatformFeed')
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 181620615..90b351cbb 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -343,7 +343,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
     def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}, account_id=None):
         real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query)
         entry = self._download_json(real_url, video_id)['entries'][0]
-        main_smil_url = 'http://link.theplatform.com/s/%s/media/guid/%d/%s' % (provider_id, account_id, entry['guid']) if account_id else None
+        main_smil_url = 'http://link.theplatform.com/s/%s/media/guid/%d/%s' % (provider_id, account_id, entry['guid']) if account_id else entry.get('plmedia$publicUrl')
 
         formats = []
         subtitles = {}
@@ -356,7 +356,8 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             if first_video_id is None:
                 first_video_id = cur_video_id
                 duration = float_or_none(item.get('plfile$duration'))
-            for asset_type in item['plfile$assetTypes']:
+            file_asset_types = item.get('plfile$assetTypes') or compat_parse_qs(compat_urllib_parse_urlparse(smil_url).query)['assetTypes']
+            for asset_type in file_asset_types:
                 if asset_type in asset_types:
                     continue
                 asset_types.append(asset_type)

From 4e33e0792a3e134b494bd71f257a674294cca8d9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 12:00:50 +0100
Subject: [PATCH 190/510] [loc] update test

---
 youtube_dl/extractor/libraryofcongress.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
index 1e5c82c66..03f205144 100644
--- a/youtube_dl/extractor/libraryofcongress.py
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -20,12 +20,11 @@ class LibraryOfCongressIE(InfoExtractor):
     _TESTS = [{
         # embedded via <div class="media-player"
         'url': 'http://loc.gov/item/90716351/',
-        'md5': '353917ff7f0255aa6d4b80a034833de8',
+        'md5': '6ec0ae8f07f86731b1b2ff70f046210a',
         'info_dict': {
             'id': '90716351',
             'ext': 'mp4',
             'title': "Pa's trip to Mars",
-            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 0,
             'view_count': int,
         },

From 6866f2449437eeb0ad93b80e5bf39cf758af7a26 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 12:08:46 +0100
Subject: [PATCH 191/510] [foxsports] update test

---
 youtube_dl/extractor/foxsports.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index 596fded20..2b2cb6c6f 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -1,10 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    smuggle_url,
-    update_url_query,
-)
 
 
 class FoxSportsIE(InfoExtractor):
@@ -14,14 +10,19 @@ class FoxSportsIE(InfoExtractor):
         'url': 'http://www.foxsports.com/tennessee/video/432609859715',
         'md5': 'b49050e955bebe32c301972e4012ac17',
         'info_dict': {
-            'id': 'bwduI3X_TgUB',
+            'id': '432609859715',
             'ext': 'mp4',
             'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
             'description': 'Courtney Lee talks about Memphis being focused.',
-            'upload_date': '20150423',
-            'timestamp': 1429761109,
+            # TODO: fix timestamp
+            'upload_date': '19700101',  # '20150423',
+            # 'timestamp': 1429761109,
             'uploader': 'NEWA-FNG-FOXSPORTS',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         'add_ie': ['ThePlatform'],
     }
 

From a843464a7e0608b679651f913cbd9447a7b928c0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 12:10:06 +0100
Subject: [PATCH 192/510] [nbc] fix NBCNews article extraction(closes #16194)

---
 youtube_dl/extractor/nbc.py | 91 ++++++++-----------------------------
 1 file changed, 19 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 765c46fd2..3282f84ee 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -9,10 +9,8 @@ from .theplatform import ThePlatformIE
 from .adobepass import AdobePassIE
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
-    find_xpath_attr,
     smuggle_url,
     try_get,
-    unescapeHTML,
     update_url_query,
     int_or_none,
 )
@@ -269,27 +267,14 @@ class CSNNEIE(InfoExtractor):
 
 
 class NBCNewsIE(ThePlatformIE):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/
-        (?:video/.+?/(?P<id>\d+)|
-        ([^/]+/)*(?:.*-)?(?P<mpx_id>[^/?]+))
-        '''
+    _VALID_URL = r'(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/([^/]+/)*(?:.*-)?(?P<id>[^/?]+)'
 
     _TESTS = [
-        {
-            'url': 'http://www.nbcnews.com/video/nbc-news/52753292',
-            'md5': '47abaac93c6eaf9ad37ee6c4463a5179',
-            'info_dict': {
-                'id': '52753292',
-                'ext': 'flv',
-                'title': 'Crew emerges after four-month Mars food study',
-                'description': 'md5:24e632ffac72b35f8b67a12d1b6ddfc1',
-            },
-        },
         {
             'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
             'md5': 'af1adfa51312291a017720403826bb64',
             'info_dict': {
-                'id': 'p_tweet_snow_140529',
+                'id': '269389891880',
                 'ext': 'mp4',
                 'title': 'How Twitter Reacted To The Snowden Interview',
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
@@ -313,7 +298,7 @@ class NBCNewsIE(ThePlatformIE):
             'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
             'md5': '73135a2e0ef819107bbb55a5a9b2a802',
             'info_dict': {
-                'id': 'nn_netcast_150204',
+                'id': '394064451844',
                 'ext': 'mp4',
                 'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
@@ -326,7 +311,7 @@ class NBCNewsIE(ThePlatformIE):
             'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
             'md5': 'a49e173825e5fcd15c13fc297fced39d',
             'info_dict': {
-                'id': 'x_lon_vwhorn_150922',
+                'id': '529953347624',
                 'ext': 'mp4',
                 'title': 'Volkswagen U.S. Chief:\xa0 We Have Totally Screwed Up',
                 'description': 'md5:c8be487b2d80ff0594c005add88d8351',
@@ -339,7 +324,7 @@ class NBCNewsIE(ThePlatformIE):
             'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
             'md5': '118d7ca3f0bea6534f119c68ef539f71',
             'info_dict': {
-                'id': 'tdy_al_space_160420',
+                'id': '669831235788',
                 'ext': 'mp4',
                 'title': 'See the aurora borealis from space in stunning new NASA video',
                 'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
@@ -352,7 +337,7 @@ class NBCNewsIE(ThePlatformIE):
             'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
             'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
             'info_dict': {
-                'id': 'n_hayes_Aimm_140801_272214',
+                'id': '314487875924',
                 'ext': 'mp4',
                 'title': 'The chaotic GOP immigration vote',
                 'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
@@ -374,60 +359,22 @@ class NBCNewsIE(ThePlatformIE):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        if video_id is not None:
-            all_info = self._download_xml('http://www.nbcnews.com/id/%s/displaymode/1219' % video_id, video_id)
-            info = all_info.find('video')
-
-            return {
-                'id': video_id,
-                'title': info.find('headline').text,
-                'ext': 'flv',
-                'url': find_xpath_attr(info, 'media', 'type', 'flashVideo').text,
-                'description': info.find('caption').text,
-                'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
-            }
-        else:
-            # "feature" and "nightly-news" pages use theplatform.com
-            video_id = mobj.group('mpx_id')
+        video_id = self._match_id(url)
+        if not video_id.isdigit():
             webpage = self._download_webpage(url, video_id)
 
-            filter_param = 'byId'
-            bootstrap_json = self._search_regex(
-                [r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
-                 r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"',
-                 r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);'],
-                webpage, 'bootstrap json', default=None)
-            if bootstrap_json:
-                bootstrap = self._parse_json(
-                    bootstrap_json, video_id, transform_source=unescapeHTML)
+            data = self._parse_json(self._search_regex(
+                r'window\.__data\s*=\s*({.+});', webpage,
+                'bootstrap json'), video_id)
+            video_id = data['article']['content'][0]['primaryMedia']['video']['mpxMetadata']['id']
 
-                info = None
-                if 'results' in bootstrap:
-                    info = bootstrap['results'][0]['video']
-                elif 'video' in bootstrap:
-                    info = bootstrap['video']
-                elif 'msnbcVideoInfo' in bootstrap:
-                    info = bootstrap['msnbcVideoInfo']['meta']
-                elif 'msnbcThePlatform' in bootstrap:
-                    info = bootstrap['msnbcThePlatform']['videoPlayer']['video']
-                else:
-                    info = bootstrap
-
-                if 'guid' in info:
-                    video_id = info['guid']
-                    filter_param = 'byGuid'
-                elif 'mpxId' in info:
-                    video_id = info['mpxId']
-
-            return {
-                '_type': 'url_transparent',
-                'id': video_id,
-                # http://feed.theplatform.com/f/2E2eJC/nbcnews also works
-                'url': update_url_query('http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews', {filter_param: video_id}),
-                'ie_key': 'ThePlatformFeed',
-            }
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            # http://feed.theplatform.com/f/2E2eJC/nbcnews also works
+            'url': update_url_query('http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews', {'byId': video_id}),
+            'ie_key': 'ThePlatformFeed',
+        }
 
 
 class NBCOlympicsIE(InfoExtractor):

From af60e81e3c557ace943aab35c1364d3d03d5a3bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Nov 2018 02:00:38 +0700
Subject: [PATCH 193/510] [setup.py] Add more relevant classifiers

---
 setup.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/setup.py b/setup.py
index 7dbb5805f..a1a08f1e2 100644
--- a/setup.py
+++ b/setup.py
@@ -124,6 +124,8 @@ setup(
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'License :: Public Domain',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
@@ -132,6 +134,12 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: Implementation',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: IronPython',
+        'Programming Language :: Python :: Implementation :: Jython',
+        'Programming Language :: Python :: Implementation :: PyPy',
     ],
 
     cmdclass={'build_lazy_extractors': build_lazy_extractors},

From bd2d553c7b1529f793c2b7343c514a558543fc0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Nov 2018 02:01:22 +0700
Subject: [PATCH 194/510] [travis] Add python 3.7 build

---
 .travis.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 92f326860..1ea640071 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,12 @@ env:
   - YTDL_TEST_SET=download
 matrix:
   include:
+    - python: 3.7
+      dist: xenial
+      env: YTDL_TEST_SET=core
+    - python: 3.7
+      dist: xenial
+      env: YTDL_TEST_SET=download
     - env: JYTHON=true; YTDL_TEST_SET=core
     - env: JYTHON=true; YTDL_TEST_SET=download
   fast_finish: true

From 157eef3e635230cbba0dd0c74f7115029867533e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Nov 2018 02:08:41 +0700
Subject: [PATCH 195/510] [setup.py] Add python 3.8 classifier

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index a1a08f1e2..dfb669ad2 100644
--- a/setup.py
+++ b/setup.py
@@ -135,6 +135,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: IronPython',

From 305ce767d586e8796d873270abf771e69ff5586c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Nov 2018 02:34:35 +0700
Subject: [PATCH 196/510] [travis] Add python 3.8-dev build

---
 .travis.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 1ea640071..79287ccf6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,6 +21,12 @@ matrix:
     - python: 3.7
       dist: xenial
       env: YTDL_TEST_SET=download
+    - python: 3.8-dev
+      dist: xenial
+      env: YTDL_TEST_SET=core
+    - python: 3.8-dev
+      dist: xenial
+      env: YTDL_TEST_SET=download
     - env: JYTHON=true; YTDL_TEST_SET=core
     - env: JYTHON=true; YTDL_TEST_SET=download
   fast_finish: true

From 560020da3049bec19e5714e9e24fc90fadd06582 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 23:19:46 +0100
Subject: [PATCH 197/510] [mixcloud] fallback to hardcoded decryption
 key(closes #18016)

---
 youtube_dl/extractor/mixcloud.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index b7bccb504..a2d19d3ef 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -161,11 +161,17 @@ class MixcloudIE(InfoExtractor):
             stream_info = info_json['streamInfo']
             formats = []
 
+            def decrypt_url(f_url):
+                for k in (key, 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'):
+                    decrypted_url = self._decrypt_xor_cipher(k, compat_b64decode(f_url))
+                    if re.search(r'^https?://[0-9a-z.]+/[0-9A-Za-z/.?=&_-]+$', decrypted_url):
+                        return decrypted_url
+
             for url_key in ('url', 'hlsUrl', 'dashUrl'):
                 format_url = stream_info.get(url_key)
                 if not format_url:
                     continue
-                decrypted = self._decrypt_xor_cipher(key, compat_b64decode(format_url))
+                decrypted = decrypt_url(format_url)
                 if not decrypted:
                     continue
                 if url_key == 'hlsUrl':

From 6f2883a2df45ca89d272bc8a0975f09758af5eb3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Nov 2018 23:25:38 +0100
Subject: [PATCH 198/510] [mixcloud] base64 decode before decryption

---
 youtube_dl/extractor/mixcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index a2d19d3ef..bcac13ec5 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -163,7 +163,7 @@ class MixcloudIE(InfoExtractor):
 
             def decrypt_url(f_url):
                 for k in (key, 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'):
-                    decrypted_url = self._decrypt_xor_cipher(k, compat_b64decode(f_url))
+                    decrypted_url = self._decrypt_xor_cipher(k, f_url)
                     if re.search(r'^https?://[0-9a-z.]+/[0-9A-Za-z/.?=&_-]+$', decrypted_url):
                         return decrypted_url
 
@@ -171,7 +171,7 @@ class MixcloudIE(InfoExtractor):
                 format_url = stream_info.get(url_key)
                 if not format_url:
                     continue
-                decrypted = decrypt_url(format_url)
+                decrypted = decrypt_url(compat_b64decode(format_url))
                 if not decrypted:
                     continue
                 if url_key == 'hlsUrl':

From 66173211c4177d36612486acfd99fc4634b8004e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Nov 2018 00:14:43 +0700
Subject: [PATCH 199/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 0083c4631..beb002041 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
++ [setup.py] Add more relevant classifiers
+
+Extractors
+* [mixcloud] Fallback to hardcoded decryption key (#18016)
+* [nbc:news] Fix article extraction (#16194)
+* [foxsports] Fix extraction (#17543)
+* [loc] Relax regular expression and improve formats extraction
++ [ciscolive] Add support for ciscolive.cisco.com (#17984)
+* [nzz] Relax kaltura regex (#18228)
+* [sixplay] Fix formats extraction
+* [bitchute] Improve title extraction
+* [kaltura] Limit requested MediaEntry fields
++ [americastestkitchen] Add support for zype embeds (#18225)
++ [pornhub] Add pornhub.net alias
+* [nova:embed] Fix extraction (#18222)
+
+
 version 2018.11.18
 
 Extractors

From d861a9d5814408973e0715bb9160fb7db34fbcd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Nov 2018 00:16:45 +0700
Subject: [PATCH 200/510] release 2018.11.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 905576364..35cc8d6d0 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.11.18
+[debug] youtube-dl version 2018.11.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index beb002041..f82c7ea35 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.11.23
 
 Core
 + [setup.py] Add more relevant classifiers
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9009f7e9e..7d72ad82d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -163,6 +163,8 @@
  - **chirbit**
  - **chirbit:profile**
  - **Cinchcast**
+ - **CiscoLiveSearch**
+ - **CiscoLiveSession**
  - **CJSW**
  - **cliphunter**
  - **Clippit**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7f5ad7bf4..4956365d0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.11.18'
+__version__ = '2018.11.23'

From 6864855eb111dbf6e0efe9ed086f48efa1d9f209 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Nov 2018 00:43:42 +0700
Subject: [PATCH 201/510] [tests] Fix invalid escape sequences

---
 test/test_compat.py         | 2 +-
 test/test_postprocessors.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index d6c54e135..51fe6aa0b 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -39,7 +39,7 @@ class TestCompat(unittest.TestCase):
 
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')
-        test_str = 'C:\Documents and Settings\тест\Application Data'
+        test_str = r'C:\Documents and Settings\тест\Application Data'
         compat_setenv('HOME', test_str)
         self.assertEqual(compat_expanduser('~'), test_str)
         compat_setenv('HOME', old_home or '')
diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
index addb69d6f..4209d1d9a 100644
--- a/test/test_postprocessors.py
+++ b/test/test_postprocessors.py
@@ -14,4 +14,4 @@ from youtube_dl.postprocessor import MetadataFromTitlePP
 class TestMetadataFromTitle(unittest.TestCase):
     def test_format_to_regex(self):
         pp = MetadataFromTitlePP(None, '%(title)s - %(artist)s')
-        self.assertEqual(pp._titleregex, '(?P<title>.+)\ \-\ (?P<artist>.+)')
+        self.assertEqual(pp._titleregex, r'(?P<title>.+)\ \-\ (?P<artist>.+)')

From 641e86e3cf751f1050ca331b9d13152bd0e18558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Nov 2018 21:47:41 +0700
Subject: [PATCH 202/510] [wistia] Add support for fast.wistia.com (closes
 #18287)

---
 youtube_dl/extractor/wistia.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 2182d6fd4..01a51275e 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class WistiaIE(InfoExtractor):
-    _VALID_URL = r'(?:wistia:|https?://(?:fast\.)?wistia\.net/embed/iframe/)(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'(?:wistia:|https?://(?:fast\.)?wistia\.(?:net|com)/embed/iframe/)(?P<id>[a-z0-9]+)'
     _API_URL = 'http://fast.wistia.com/embed/medias/%s.json'
     _IFRAME_URL = 'http://fast.wistia.net/embed/iframe/%s'
 
@@ -35,12 +35,15 @@ class WistiaIE(InfoExtractor):
         # with hls video
         'url': 'wistia:807fafadvk',
         'only_matching': True,
+    }, {
+        'url': 'http://fast.wistia.com/embed/iframe/sh7fpupwlt',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_url(webpage):
         match = re.search(
-            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
+            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.(?:net|com)/embed/iframe/.+?)\1', webpage)
         if match:
             return unescapeHTML(match.group('url'))
 

From d19600df07128c73ef7242af7e1cd8c819951aba Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Sat, 24 Nov 2018 16:14:27 +0100
Subject: [PATCH 203/510] [joj] Fix extraction (closes #18280)

---
 youtube_dl/extractor/joj.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/joj.py b/youtube_dl/extractor/joj.py
index d9f8dbfd2..62b28e980 100644
--- a/youtube_dl/extractor/joj.py
+++ b/youtube_dl/extractor/joj.py
@@ -61,7 +61,7 @@ class JojIE(InfoExtractor):
 
         bitrates = self._parse_json(
             self._search_regex(
-                r'(?s)bitrates\s*=\s*({.+?});', webpage, 'bitrates',
+                r'(?s)(?:src|bitrates)\s*=\s*({.+?});', webpage, 'bitrates',
                 default='{}'),
             video_id, transform_source=js_to_json, fatal=False)
 

From ca01d178844129bd4b6ed74740fbd30e7f84c1c2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 28 Nov 2018 19:53:22 +0100
Subject: [PATCH 204/510] [vimeo] Add support for VHX(Vimeo OTT)(#14835)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vimeo.py      | 85 +++++++++++++++++++++++++++++-
 2 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 60e6175b1..cd91c0fcb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1303,6 +1303,7 @@ from .vimeo import (
     VimeoReviewIE,
     VimeoUserIE,
     VimeoWatchLaterIE,
+    VHXEmbedIE,
 )
 from .vimple import VimpleIE
 from .vine import (
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 88f4d9979..6353c6831 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -14,10 +14,13 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     ExtractorError,
+    js_to_json,
     InAdvancePagedList,
     int_or_none,
     merge_dicts,
     NO_DEFAULT,
+    parse_filesize,
+    qualities,
     RegexNotFoundError,
     sanitized_Request,
     smuggle_url,
@@ -27,7 +30,6 @@ from ..utils import (
     unsmuggle_url,
     urlencode_postdata,
     unescapeHTML,
-    parse_filesize,
 )
 
 
@@ -1063,3 +1065,84 @@ class VimeoLikesIE(InfoExtractor):
             'description': description,
             'entries': pl,
         }
+
+
+class VHXEmbedIE(InfoExtractor):
+    IE_NAME = 'vhx:embed'
+    _VALID_URL = r'https?://embed\.vhx\.tv/videos/(?P<id>\d+)'
+
+    def _call_api(self, video_id, access_token, path='', query=None):
+        return self._download_json(
+            'https://api.vhx.tv/videos/' + video_id + path, video_id, headers={
+                'Authorization': 'Bearer ' + access_token,
+            }, query=query)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        credentials = self._parse_json(self._search_regex(
+            r'(?s)credentials\s*:\s*({.+?}),', webpage,
+            'config'), video_id, js_to_json)
+        access_token = credentials['access_token']
+
+        query = {}
+        for k, v in credentials.items():
+            if k in ('authorization', 'authUserToken', 'ticket') and v and v != 'undefined':
+                if k == 'authUserToken':
+                    query['auth_user_token'] = v
+                else:
+                    query[k] = v
+        files = self._call_api(video_id, access_token, '/files', query)
+
+        formats = []
+        for f in files:
+            href = try_get(f, lambda x: x['_links']['source']['href'])
+            if not href:
+                continue
+            method = f.get('method')
+            if method == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    href, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif method == 'dash':
+                formats.extend(self._extract_mpd_formats(
+                    href, video_id, mpd_id='dash', fatal=False))
+            else:
+                fmt = {
+                    'filesize': int_or_none(try_get(f, lambda x: x['size']['bytes'])),
+                    'format_id': 'http',
+                    'preference': 1,
+                    'url': href,
+                    'vcodec': f.get('codec'),
+                }
+                quality = f.get('quality')
+                if quality:
+                    fmt.update({
+                        'format_id': 'http-' + quality,
+                        'height': int_or_none(self._search_regex(r'(\d+)p', quality, 'height', default=None)),
+                    })
+                formats.append(fmt)
+        self._sort_formats(formats)
+
+        video_data = self._call_api(video_id, access_token)
+        title = video_data.get('title') or video_data['name']
+
+        q = qualities(['small', 'medium', 'large', 'source'])
+        thumbnails = []
+        for thumbnail_id, thumbnail_url in video_data.get('thumbnail', {}).items():
+            thumbnails.append({
+                'id': thumbnail_id,
+                'url': thumbnail_url,
+                'preference': q(thumbnail_id),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'duration': int_or_none(try_get(video_data, lambda x: x['duration']['seconds'])),
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'timestamp': unified_timestamp(video_data.get('created_at')),
+            'view_count': int_or_none(video_data.get('plays_count')),
+        }

From d9df8f120b325766181fb474a8c534e51df78f17 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 28 Nov 2018 20:13:36 +0100
Subject: [PATCH 205/510] [vimeo] extract VHX subtitles

---
 youtube_dl/extractor/vimeo.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6353c6831..5e15f060b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -1127,6 +1127,17 @@ class VHXEmbedIE(InfoExtractor):
         video_data = self._call_api(video_id, access_token)
         title = video_data.get('title') or video_data['name']
 
+        subtitles = {}
+        for subtitle in try_get(video_data, lambda x: x['tracks']['subtitles'], list) or []:
+            lang = subtitle.get('srclang') or subtitle.get('label')
+            for _link in subtitle.get('_links', {}).values():
+                href = _link.get('href')
+                if not href:
+                    continue
+                subtitles.setdefault(lang, []).append({
+                    'url': href,
+                })
+
         q = qualities(['small', 'medium', 'large', 'source'])
         thumbnails = []
         for thumbnail_id, thumbnail_url in video_data.get('thumbnail', {}).items():
@@ -1142,6 +1153,7 @@ class VHXEmbedIE(InfoExtractor):
             'description': video_data.get('description'),
             'duration': int_or_none(try_get(video_data, lambda x: x['duration']['seconds'])),
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnails': thumbnails,
             'timestamp': unified_timestamp(video_data.get('created_at')),
             'view_count': int_or_none(video_data.get('plays_count')),

From 053e5b12b2e38b7d343aafbb7dc13fb8e4933015 Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Thu, 29 Nov 2018 18:12:18 +0100
Subject: [PATCH 206/510] [azmedien] Fix extraction (closes #18334)

---
 youtube_dl/extractor/azmedien.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index a57a5f114..fcbdc71b9 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -36,7 +36,6 @@ class AZMedienIE(InfoExtractor):
             'id': '1_anruz3wy',
             'ext': 'mp4',
             'title': 'Bundesrats-Vakanzen / EU-Rahmenabkommen',
-            'description': 'md5:dd9f96751ec9c35e409a698a328402f3',
             'uploader_id': 'TVOnline',
             'upload_date': '20180930',
             'timestamp': 1538328802,
@@ -53,15 +52,12 @@ class AZMedienIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        host = mobj.group('host')
         video_id = mobj.group('id')
         entry_id = mobj.group('kaltura_id')
 
         if not entry_id:
-            webpage = self._download_webpage(url, video_id)
-            api_path = self._search_regex(
-                r'["\']apiPath["\']\s*:\s*["\']([^"^\']+)["\']',
-                webpage, 'api path')
-            api_url = 'https://www.%s%s' % (mobj.group('host'), api_path)
+            api_url = 'https://www.%s/api/pub/gql/%s' % (host, host.split('.')[0])
             payload = {
                 'query': '''query VideoContext($articleId: ID!) {
                     article: node(id: $articleId) {

From adbbdefc8126a933d9ff0a6e603fb312e4b4cbdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Nov 2018 00:48:15 +0700
Subject: [PATCH 207/510] [hotstar] Add support for alternative app state
 layout (closes #18320)

---
 youtube_dl/extractor/hotstar.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index bf5717f1b..45aa5e7ea 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -43,6 +43,7 @@ class HotStarIE(HotStarBaseIE):
     IE_NAME = 'hotstar'
     _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
     _TESTS = [{
+        # contentData
         'url': 'https://www.hotstar.com/can-you-not-spread-rumours/1000076273',
         'info_dict': {
             'id': '1000076273',
@@ -57,6 +58,10 @@ class HotStarIE(HotStarBaseIE):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        # contentDetail
+        'url': 'https://www.hotstar.com/movies/radha-gopalam/1000057157',
+        'only_matching': True,
     }, {
         'url': 'http://www.hotstar.com/sports/cricket/rajitha-sizzles-on-debut-with-329/2001477583',
         'only_matching': True,
@@ -74,8 +79,12 @@ class HotStarIE(HotStarBaseIE):
             r'<script>window\.APP_STATE\s*=\s*({.+?})</script>',
             webpage, 'app state'), video_id)
         video_data = {}
+        getters = (
+            lambda x, k=k: x['initialState']['content%s' % k]['content']
+            for k in ('Data', 'Detail')
+        )
         for v in app_state.values():
-            content = try_get(v, lambda x: x['initialState']['contentData']['content'], dict)
+            content = try_get(v, getters, dict)
             if content and content.get('contentId') == video_id:
                 video_data = content
 

From 16597c2f9492651c55e01f441ee9cb5c276209cb Mon Sep 17 00:00:00 2001
From: Jimm Stout <jamesstout1@gmail.com>
Date: Thu, 29 Nov 2018 13:07:07 -0500
Subject: [PATCH 208/510] [gfycat] Update API endpoint (closes #18333)

---
 youtube_dl/extractor/gfycat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index a0670b645..c1b36a59b 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -53,7 +53,7 @@ class GfycatIE(InfoExtractor):
         video_id = self._match_id(url)
 
         gfy = self._download_json(
-            'http://gfycat.com/cajax/get/%s' % video_id,
+            'https://api.gfycat.com/v1/gfycats/%s' % video_id,
             video_id, 'Downloading video info')
         if 'error' in gfy:
             raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)

From f012823082c893c0fc1f96afb8a91f5b1c1ae07a Mon Sep 17 00:00:00 2001
From: Hakim Boyles <hak@volkanite.net>
Date: Thu, 29 Nov 2018 14:20:27 -0400
Subject: [PATCH 209/510] [lynda] Fix authentication (closes #18158)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 4ba61cd8a..3084c6dff 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class LyndaBaseIE(InfoExtractor):
-    _SIGNIN_URL = 'https://www.lynda.com/signin'
+    _SIGNIN_URL = 'https://www.lynda.com/signin/lynda'
     _PASSWORD_URL = 'https://www.lynda.com/signin/password'
     _USER_URL = 'https://www.lynda.com/signin/user'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'

From 3430ff9b07d4dc9dd39617af54aeffb381d88737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Dec 2018 16:45:51 +0700
Subject: [PATCH 210/510] [pornhub] Use actual URL host for requests (closes
 #18359)

---
 youtube_dl/extractor/pornhub.py | 34 +++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index c9c884095..e377de196 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -27,7 +27,7 @@ class PornHubIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:[^/]+\.)?pornhub\.(?:com|net)/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
+                            (?:[^/]+\.)?(?P<host>pornhub\.(?:com|net))/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
                             (?:www\.)?thumbzilla\.com/video/
                         )
                         (?P<id>[\da-z]+)
@@ -129,7 +129,7 @@ class PornHubIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
-            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub\.com/embed/[\da-z]+)',
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub\.(?:com|net)/embed/[\da-z]+)',
             webpage)
 
     def _extract_count(self, pattern, webpage, name):
@@ -137,14 +137,16 @@ class PornHubIE(InfoExtractor):
             pattern, webpage, '%s count' % name, fatal=False))
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        host = mobj.group('host') or 'pornhub.com'
+        video_id = mobj.group('id')
 
-        self._set_cookie('pornhub.com', 'age_verified', '1')
+        self._set_cookie(host, 'age_verified', '1')
 
         def dl_webpage(platform):
-            self._set_cookie('pornhub.com', 'platform', platform)
+            self._set_cookie(host, 'platform', platform)
             return self._download_webpage(
-                'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id,
+                'http://www.%s/view_video.php?viewkey=%s' % (host, video_id),
                 video_id, 'Downloading %s webpage' % platform)
 
         webpage = dl_webpage('pc')
@@ -306,7 +308,7 @@ class PornHubIE(InfoExtractor):
 
 
 class PornHubPlaylistBaseIE(InfoExtractor):
-    def _extract_entries(self, webpage):
+    def _extract_entries(self, webpage, host):
         # Only process container div with main playlist content skipping
         # drop-down menu that uses similar pattern for videos (see
         # https://github.com/rg3/youtube-dl/issues/11594).
@@ -316,7 +318,7 @@ class PornHubPlaylistBaseIE(InfoExtractor):
 
         return [
             self.url_result(
-                'http://www.pornhub.com/%s' % video_url,
+                'http://www.%s/%s' % (host, video_url),
                 PornHubIE.ie_key(), video_title=title)
             for video_url, title in orderedSet(re.findall(
                 r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
@@ -324,11 +326,13 @@ class PornHubPlaylistBaseIE(InfoExtractor):
         ]
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        host = mobj.group('host')
+        playlist_id = mobj.group('id')
 
         webpage = self._download_webpage(url, playlist_id)
 
-        entries = self._extract_entries(webpage)
+        entries = self._extract_entries(webpage, host)
 
         playlist = self._parse_json(
             self._search_regex(
@@ -343,7 +347,7 @@ class PornHubPlaylistBaseIE(InfoExtractor):
 
 
 class PornHubPlaylistIE(PornHubPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.(?:com|net)/playlist/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?P<host>pornhub\.(?:com|net))/playlist/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/playlist/4667351',
         'info_dict': {
@@ -358,7 +362,7 @@ class PornHubPlaylistIE(PornHubPlaylistBaseIE):
 
 
 class PornHubUserVideosIE(PornHubPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:[^/]+\.)?pornhub\.(?:com|net)/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos'
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?P<host>pornhub\.(?:com|net))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos'
     _TESTS = [{
         'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
         'info_dict': {
@@ -399,7 +403,9 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
     }]
 
     def _real_extract(self, url):
-        user_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        host = mobj.group('host')
+        user_id = mobj.group('id')
 
         entries = []
         for page_num in itertools.count(1):
@@ -411,7 +417,7 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
                     break
                 raise
-            page_entries = self._extract_entries(webpage)
+            page_entries = self._extract_entries(webpage, host)
             if not page_entries:
                 break
             entries.extend(page_entries)

From aa374bc78e5e4dbb8453bd367ae0e3c0db702a7a Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Sat, 1 Dec 2018 18:05:15 +0100
Subject: [PATCH 211/510] [utils] Fix random_birthday to generate existing
 dates only

---
 youtube_dl/utils.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e84d35d4d..0b1c7cd6c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3948,8 +3948,12 @@ def write_xattr(path, key, value):
 
 
 def random_birthday(year_field, month_field, day_field):
+    start_date = datetime.date(1950, 1, 1)
+    end_date = datetime.date(1995, 12, 31)
+    offset = random.randint(0, (end_date - start_date).days)
+    random_date = start_date + datetime.timedelta(offset)
     return {
-        year_field: str(random.randint(1950, 1995)),
-        month_field: str(random.randint(1, 12)),
-        day_field: str(random.randint(1, 31)),
+        year_field: str(random_date.year),
+        month_field: str(random_date.month),
+        day_field: str(random_date.day),
     }

From 1ead840d2c18d4add340117f676fd6694f0650d3 Mon Sep 17 00:00:00 2001
From: Ken Swenson <flat@esoteric.moe>
Date: Fri, 9 Nov 2018 16:49:20 -0500
Subject: [PATCH 212/510] [tiktok] Add extractor (closes #18108)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tiktok.py     | 79 ++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tiktok.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index cd91c0fcb..547331078 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1124,6 +1124,7 @@ from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
 from .threeqsdn import ThreeQSDNIE
+from .tiktok import TikTokIE
 from .tinypic import TinyPicIE
 from .tmz import (
     TMZIE,
diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
new file mode 100644
index 000000000..d71b09c66
--- /dev/null
+++ b/youtube_dl/extractor/tiktok.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_str,
+    int_or_none,
+    str_or_none,
+    try_get,
+    url_or_none,
+)
+
+
+class TikTokIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:m\.)?tiktok\.com/v/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://m.tiktok.com/v/6606727368545406213.html',
+        'md5': 'd584b572e92fcd48888051f238022420',
+        'info_dict': {
+            'id': '6606727368545406213',
+            'ext': 'mp4',
+            'title': 'Zureeal on TikTok',
+            'thumbnail': r're:^https?://.*~noop.image',
+            'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay',
+            'uploader': 'Zureeal',
+            'width': 540,
+            'height': 960,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        data = self._parse_json(
+            self._search_regex(
+                r'var\s+data\s*=\s*({.+?});', webpage, 'data'
+            ), video_id)
+
+        title = self._og_search_title(webpage)
+
+        description = str_or_none(try_get(data, lambda x: x['desc']))
+        width = int_or_none(try_get(data, lambda x: x['video']['width']))
+        height = int_or_none(try_get(data, lambda x: x['video']['height']))
+
+        formats = []
+
+        for count, (key, label) in enumerate((('play_addr_lowbr', 'Low'), ('play_addr', 'Normal'), ('download_addr', 'Download')), -2):
+            for format in try_get(data, lambda x: x['video'][key]['url_list']):
+                format_url = url_or_none(format)
+                if not format_url:
+                    continue
+                formats.append({
+                    'url': format_url,
+                    'ext': 'mp4',
+                    'height': height,
+                    'width': width,
+                    'format_note': label,
+                    'quality': count
+                })
+
+        self._sort_formats(formats)
+
+        uploader = try_get(data, lambda x: x['author']['nickname'], compat_str)
+
+        thumbnail = url_or_none(
+            try_get(
+                data, lambda x: x['video']['cover']['url_list'][0], compat_str))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'uploader': uploader,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'width': width,
+            'height': height,
+        }

From ce18a19be9af1e227e7162636cbf6b277adc1b41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Dec 2018 02:39:22 +0700
Subject: [PATCH 213/510] [tiktok] Improve extraction and add support for user
 pages (closes #18135)

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/tiktok.py     | 136 ++++++++++++++++++-----------
 2 files changed, 91 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 547331078..0baed6b27 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1124,7 +1124,10 @@ from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
 from .threeqsdn import ThreeQSDNIE
-from .tiktok import TikTokIE
+from .tiktok import (
+    TikTokIE,
+    TikTokUserIE,
+)
 from .tinypic import TinyPicIE
 from .tmz import (
     TMZIE,
diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index d71b09c66..083e9f36d 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     compat_str,
+    ExtractorError,
     int_or_none,
     str_or_none,
     try_get,
@@ -11,69 +12,106 @@ from ..utils import (
 )
 
 
-class TikTokIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tiktok\.com/v/(?P<id>[0-9]+)'
+class TikTokBaseIE(InfoExtractor):
+    def _extract_aweme(self, data):
+        video = data['video']
+        description = str_or_none(try_get(data, lambda x: x['desc']))
+        width = int_or_none(try_get(data, lambda x: video['width']))
+        height = int_or_none(try_get(data, lambda x: video['height']))
+
+        format_urls = set()
+        formats = []
+        for format_id in (
+                'play_addr_lowbr', 'play_addr', 'play_addr_h264',
+                'download_addr'):
+            for format in try_get(
+                    video, lambda x: x[format_id]['url_list'], list) or []:
+                format_url = url_or_none(format)
+                if not format_url:
+                    continue
+                if format_url in format_urls:
+                    continue
+                format_urls.add(format_url)
+                formats.append({
+                    'url': format_url,
+                    'ext': 'mp4',
+                    'height': height,
+                    'width': width,
+                })
+        self._sort_formats(formats)
+
+        thumbnail = url_or_none(try_get(
+            video, lambda x: x['cover']['url_list'][0], compat_str))
+        uploader = try_get(data, lambda x: x['author']['nickname'], compat_str)
+        timestamp = int_or_none(data.get('create_time'))
+        comment_count = int_or_none(data.get('comment_count')) or int_or_none(
+            try_get(data, lambda x: x['statistics']['comment_count']))
+        repost_count = int_or_none(try_get(
+            data, lambda x: x['statistics']['share_count']))
+
+        aweme_id = data['aweme_id']
+
+        return {
+            'id': aweme_id,
+            'title': uploader or aweme_id,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'timestamp': timestamp,
+            'comment_count': comment_count,
+            'repost_count': repost_count,
+            'formats': formats,
+        }
+
+
+class TikTokIE(TikTokBaseIE):
+    _VALID_URL = r'https?://(?:m\.)?tiktok\.com/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://m.tiktok.com/v/6606727368545406213.html',
         'md5': 'd584b572e92fcd48888051f238022420',
         'info_dict': {
             'id': '6606727368545406213',
             'ext': 'mp4',
-            'title': 'Zureeal on TikTok',
-            'thumbnail': r're:^https?://.*~noop.image',
+            'title': 'Zureeal',
             'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay',
+            'thumbnail': r're:^https?://.*~noop.image',
             'uploader': 'Zureeal',
-            'width': 540,
-            'height': 960,
+            'timestamp': 1538248586,
+            'upload_date': '20180929',
+            'comment_count': int,
+            'repost_count': int,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        data = self._parse_json(self._search_regex(
+            r'\bdata\s*=\s*({.+?})\s*;', webpage, 'data'), video_id)
+        return self._extract_aweme(data)
 
-        data = self._parse_json(
-            self._search_regex(
-                r'var\s+data\s*=\s*({.+?});', webpage, 'data'
-            ), video_id)
 
-        title = self._og_search_title(webpage)
+class TikTokUserIE(TikTokBaseIE):
+    _VALID_URL = r'https?://(?:m\.)?tiktok\.com/h5/share/usr/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://m.tiktok.com/h5/share/usr/188294915489964032.html',
+        'info_dict': {
+            'id': '188294915489964032',
+        },
+        'playlist_mincount': 24,
+    }
 
-        description = str_or_none(try_get(data, lambda x: x['desc']))
-        width = int_or_none(try_get(data, lambda x: x['video']['width']))
-        height = int_or_none(try_get(data, lambda x: x['video']['height']))
-
-        formats = []
-
-        for count, (key, label) in enumerate((('play_addr_lowbr', 'Low'), ('play_addr', 'Normal'), ('download_addr', 'Download')), -2):
-            for format in try_get(data, lambda x: x['video'][key]['url_list']):
-                format_url = url_or_none(format)
-                if not format_url:
-                    continue
-                formats.append({
-                    'url': format_url,
-                    'ext': 'mp4',
-                    'height': height,
-                    'width': width,
-                    'format_note': label,
-                    'quality': count
-                })
-
-        self._sort_formats(formats)
-
-        uploader = try_get(data, lambda x: x['author']['nickname'], compat_str)
-
-        thumbnail = url_or_none(
-            try_get(
-                data, lambda x: x['video']['cover']['url_list'][0], compat_str))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'uploader': uploader,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'width': width,
-            'height': height,
-        }
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+        data = self._download_json(
+            'https://m.tiktok.com/h5/share/usr/list/%s/' % user_id, user_id,
+            query={'_signature': '_'})
+        entries = []
+        for aweme in data['aweme_list']:
+            try:
+                entry = self._extract_aweme(aweme)
+            except ExtractorError:
+                continue
+            entry['extractor_key'] = TikTokIE.ie_key()
+            entries.append(entry)
+        return self.playlist_result(entries, user_id)

From 1fa59a928e48ac81e2afb5dfe134fa6c40f64e44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Dec 2018 00:06:54 +0700
Subject: [PATCH 214/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f82c7ea35..ddd4bec14 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Core
+* [utils] Fix random_birthday to generate existing dates only (#18284)
+
+Extractors
++ [tiktok] Add support for tiktok.com (#18108, #18135)
+* [pornhub] Use actual URL host for requests (#18359)
+* [lynda] Fix authentication (#18158, #18217)
+* [gfycat] Update API endpoint (#18333, #18343)
++ [hotstar] Add support for alternative app state layout (#18320)
+* [azmedien] Fix extraction (#18334, #18336)
++ [vimeo] Add support for VHX (Vimeo OTT) (#14835)
+* [joj] Fix extraction (#18280, #18281)
++ [wistia] Add support for fast.wistia.com (#18287)
+
+
 version 2018.11.23
 
 Core

From ab896fa894d2395b886e5bb8168ca0e0e6e9517d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Dec 2018 00:10:20 +0700
Subject: [PATCH 215/510] release 2018.12.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 35cc8d6d0..4b35244a8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.11.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.11.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.03*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.11.23
+[debug] youtube-dl version 2018.12.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ddd4bec14..689d07826 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.12.03
 
 Core
 * [utils] Fix random_birthday to generate existing dates only (#18284)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7d72ad82d..837b0199b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -885,6 +885,8 @@
  - **ThisAmericanLife**
  - **ThisAV**
  - **ThisOldHouse**
+ - **TikTok**
+ - **TikTokUser**
  - **tinypic**: tinypic.com videos
  - **TMZ**
  - **TMZArticle**
@@ -979,6 +981,7 @@
  - **VevoPlaylist**
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
+ - **vhx:embed**
  - **Viafree**
  - **vice**
  - **vice:article**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4956365d0..8e1203892 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.11.23'
+__version__ = '2018.12.03'

From 5547014ad972a4364c4d4a613db6d3a18f25950e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 2 Dec 2018 20:01:36 +0100
Subject: [PATCH 216/510] [gamespot] add support reviews URLs

---
 youtube_dl/extractor/gamespot.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index ab647dd41..4236a5ed8 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(OnceIE):
-    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article|review)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
@@ -41,6 +41,9 @@ class GameSpotIE(OnceIE):
     }, {
         'url': 'https://www.gamespot.com/articles/the-last-of-us-2-receives-new-ps4-trailer/1100-6454469/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.gamespot.com/reviews/gears-of-war-review/1900-6161188/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 8bb0c9cc16b841ab89e15e620e00ee954ab316ed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 5 Dec 2018 07:03:00 +0100
Subject: [PATCH 217/510] [tbs] fix info extraction(fixes #18403)

---
 youtube_dl/extractor/tbs.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index 784f8ed66..e8a7c65e0 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class TBSIE(TurnerBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/(?:movies|shows/[^/]+/(?:clips|season-\d+/episode-\d+))/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com(?P<path>/(?:movies|shows/[^/]+/(?:clips|season-\d+/episode-\d+))/(?P<id>[^/?#]+))'
     _TESTS = [{
         'url': 'http://www.tntdrama.com/shows/the-alienist/clips/monster',
         'info_dict': {
@@ -40,12 +40,12 @@ class TBSIE(TurnerBaseIE):
     }]
 
     def _real_extract(self, url):
-        site, display_id = re.match(self._VALID_URL, url).groups()
+        site, path, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
         drupal_settings = self._parse_json(self._search_regex(
             r'<script[^>]+?data-drupal-selector="drupal-settings-json"[^>]*?>({.+?})</script>',
             webpage, 'drupal setting'), display_id)
-        video_data = drupal_settings['turner_playlist'][0]
+        video_data = next(v for v in drupal_settings['turner_playlist'] if v.get('url') == path)
 
         media_id = video_data['mediaID']
         title = video_data['title']

From ae9d77dab54630ac480bd4082468d543a15c341e Mon Sep 17 00:00:00 2001
From: v-delta <45652398+v-delta@users.noreply.github.com>
Date: Thu, 6 Dec 2018 17:24:35 +0100
Subject: [PATCH 218/510] [yourporn] Fix extraction (closes #18424)

---
 youtube_dl/extractor/yourporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py
index 6602f7c03..a9951f3b8 100644
--- a/youtube_dl/extractor/yourporn.py
+++ b/youtube_dl/extractor/yourporn.py
@@ -26,7 +26,7 @@ class YourPornIE(InfoExtractor):
             self._search_regex(
                 r'data-vnfo=(["\'])(?P<data>{.+?})\1', webpage, 'data info',
                 group='data'),
-            video_id)[video_id])
+            video_id)[video_id]).replace('/cdn/', '/cdn2/')
 
         title = (self._search_regex(
             r'<[^>]+\bclass=["\']PostEditTA[^>]+>([^<]+)', webpage, 'title',

From 33cc1ea586480ccc60fd25f2d42cfb44eec605c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Dec 2018 00:00:06 +0700
Subject: [PATCH 219/510] [nrktv] Relax _VALID_URL (closes #18304, closes
 #18387)

---
 youtube_dl/extractor/nrk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index a231735fb..c5001ef48 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -248,7 +248,7 @@ class NRKTVIE(NRKBaseIE):
     _VALID_URL = r'''(?x)
                         https?://
                             (?:tv|radio)\.nrk(?:super)?\.no/
-                            (?:serie/[^/]+|program)/
+                            (?:serie(?:/[^/]+){1,2}|program)/
                             (?![Ee]pisodes)%s
                             (?:/\d{2}-\d{2}-\d{4})?
                             (?:\#del=(?P<part_id>\d+))?
@@ -362,6 +362,9 @@ class NRKTVIE(NRKBaseIE):
     }, {
         'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
         'only_matching': True,
+    }, {
+        'url': 'https://tv.nrk.no/serie/lindmo/2018/MUHU11006318/avspiller',
+        'only_matching': True,
     }]
 
 

From 15699ec8b0a9cb8d519b721a5cb8199afe62fc3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Dec 2018 00:49:24 +0700
Subject: [PATCH 220/510] [nrktv:season,series] Fix extraction and update tests
 (closes #17159, closes #17258)

---
 youtube_dl/extractor/nrk.py | 68 +++++++++++++++++++++++--------------
 1 file changed, 43 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index c5001ef48..48bc6fd7a 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -211,13 +211,13 @@ class NRKIE(NRKBaseIE):
     _TESTS = [{
         # video
         'url': 'http://www.nrk.no/video/PS*150533',
-        'md5': '2f7f6eeb2aacdd99885f355428715cfa',
+        'md5': '706f34cdf1322577589e369e522b50ef',
         'info_dict': {
             'id': '150533',
             'ext': 'mp4',
             'title': 'Dompap og andre fugler i Piip-Show',
             'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
-            'duration': 263,
+            'duration': 262,
         }
     }, {
         # audio
@@ -256,14 +256,14 @@ class NRKTVIE(NRKBaseIE):
     _API_HOSTS = ('psapi-ne.nrk.no', 'psapi-we.nrk.no')
     _TESTS = [{
         'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
-        'md5': '4e9ca6629f09e588ed240fb11619922a',
+        'md5': '9a167e54d04671eb6317a37b7bc8a280',
         'info_dict': {
             'id': 'MUHH48000314AA',
             'ext': 'mp4',
             'title': '20 spørsmål 23.05.2014',
             'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
             'duration': 1741,
-            'series': '20 spørsmål - TV',
+            'series': '20 spørsmål',
             'episode': '23.05.2014',
         },
     }, {
@@ -301,7 +301,7 @@ class NRKTVIE(NRKBaseIE):
                 'id': 'MSPO40010515AH',
                 'ext': 'mp4',
                 'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 1)',
-                'description': 'md5:c03aba1e917561eface5214020551b7a',
+                'description': 'md5:1f97a41f05a9486ee00c56f35f82993d',
                 'duration': 772,
                 'series': 'Tour de Ski',
                 'episode': '06.01.2015',
@@ -314,7 +314,7 @@ class NRKTVIE(NRKBaseIE):
                 'id': 'MSPO40010515BH',
                 'ext': 'mp4',
                 'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 2)',
-                'description': 'md5:c03aba1e917561eface5214020551b7a',
+                'description': 'md5:1f97a41f05a9486ee00c56f35f82993d',
                 'duration': 6175,
                 'series': 'Tour de Ski',
                 'episode': '06.01.2015',
@@ -326,7 +326,7 @@ class NRKTVIE(NRKBaseIE):
         'info_dict': {
             'id': 'MSPO40010515',
             'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
-            'description': 'md5:c03aba1e917561eface5214020551b7a',
+            'description': 'md5:1f97a41f05a9486ee00c56f35f82993d',
         },
         'expected_warnings': ['Video is geo restricted'],
     }, {
@@ -406,21 +406,35 @@ class NRKTVSerieBaseIE(InfoExtractor):
     def _extract_series(self, webpage, display_id, fatal=True):
         config = self._parse_json(
             self._search_regex(
-                r'({.+?})\s*,\s*"[^"]+"\s*\)\s*</script>', webpage, 'config',
-                default='{}' if not fatal else NO_DEFAULT),
+                (r'INITIAL_DATA_*\s*=\s*({.+?})\s*;',
+                 r'({.+?})\s*,\s*"[^"]+"\s*\)\s*</script>'),
+                webpage, 'config', default='{}' if not fatal else NO_DEFAULT),
             display_id, fatal=False)
         if not config:
             return
-        return try_get(config, lambda x: x['series'], dict)
+        return try_get(
+            config,
+            (lambda x: x['initialState']['series'], lambda x: x['series']),
+            dict)
+
+    def _extract_seasons(self, seasons):
+        if not isinstance(seasons, list):
+            return []
+        entries = []
+        for season in seasons:
+            entries.extend(self._extract_episodes(season))
+        return entries
 
     def _extract_episodes(self, season):
-        entries = []
         if not isinstance(season, dict):
-            return entries
-        episodes = season.get('episodes')
-        if not isinstance(episodes, list):
-            return entries
-        for episode in episodes:
+            return []
+        return self._extract_entries(season.get('episodes'))
+
+    def _extract_entries(self, entry_list):
+        if not isinstance(entry_list, list):
+            return []
+        entries = []
+        for episode in entry_list:
             nrk_id = episode.get('prfId')
             if not nrk_id or not isinstance(nrk_id, compat_str):
                 continue
@@ -465,7 +479,7 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
     _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/serie/(?P<id>[^/]+)'
     _ITEM_RE = r'(?:data-season=["\']|id=["\']season-)(?P<id>\d+)'
     _TESTS = [{
-        # new layout
+        # new layout, seasons
         'url': 'https://tv.nrk.no/serie/backstage',
         'info_dict': {
             'id': 'backstage',
@@ -474,20 +488,21 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
         },
         'playlist_mincount': 60,
     }, {
-        # old layout
+        # new layout, instalments
         'url': 'https://tv.nrk.no/serie/groenn-glede',
         'info_dict': {
             'id': 'groenn-glede',
             'title': 'Grønn glede',
             'description': 'md5:7576e92ae7f65da6993cf90ee29e4608',
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 10,
     }, {
-        'url': 'http://tv.nrksuper.no/serie/labyrint',
+        # old layout
+        'url': 'https://tv.nrksuper.no/serie/labyrint',
         'info_dict': {
             'id': 'labyrint',
             'title': 'Labyrint',
-            'description': 'md5:58afd450974c89e27d5a19212eee7115',
+            'description': 'md5:318b597330fdac5959247c9b69fdb1ec',
         },
         'playlist_mincount': 3,
     }, {
@@ -520,11 +535,11 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
             description = try_get(
                 series, lambda x: x['titles']['subtitle'], compat_str)
             entries = []
-            for season in series['seasons']:
-                entries.extend(self._extract_episodes(season))
+            entries.extend(self._extract_seasons(series.get('seasons')))
+            entries.extend(self._extract_entries(series.get('instalments')))
             return self.playlist_result(entries, series_id, title, description)
 
-        # Old layout (e.g. https://tv.nrk.no/serie/groenn-glede)
+        # Old layout (e.g. https://tv.nrksuper.no/serie/labyrint)
         entries = [
             self.url_result(
                 'https://tv.nrk.no/program/Episodes/{series}/{season}'.format(
@@ -536,6 +551,9 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
             'seriestitle', webpage,
             'title', default=None) or self._og_search_title(
             webpage, fatal=False)
+        if title:
+            title = self._search_regex(
+                r'NRK (?:Super )?TV\s*[-–]\s*(.+)', title, 'title', default=title)
 
         description = self._html_search_meta(
             'series_description', webpage,
@@ -596,7 +614,7 @@ class NRKPlaylistIE(NRKPlaylistBaseIE):
             'title': 'Rivertonprisen til Karin Fossum',
             'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
         },
-        'playlist_count': 5,
+        'playlist_count': 2,
     }]
 
     def _extract_title(self, webpage):

From c976873c5b2912c06ff53e5193640ee8627edee4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Dec 2018 00:54:58 +0700
Subject: [PATCH 221/510] [nrktv:series] Add support for extra materials

---
 youtube_dl/extractor/nrk.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 48bc6fd7a..072f920a9 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -537,6 +537,7 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
             entries = []
             entries.extend(self._extract_seasons(series.get('seasons')))
             entries.extend(self._extract_entries(series.get('instalments')))
+            entries.extend(self._extract_episodes(series.get('extraMaterial')))
             return self.playlist_result(entries, series_id, title, description)
 
         # Old layout (e.g. https://tv.nrksuper.no/serie/labyrint)

From dfe0a3a9d2e07ac1e5ad221912d03b999ebb4d75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Dec 2018 03:27:11 +0700
Subject: [PATCH 222/510] [lecturio] Add extractor (closes #18405)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/lecturio.py   | 186 +++++++++++++++++++++++++++++
 2 files changed, 190 insertions(+)
 create mode 100644 youtube_dl/extractor/lecturio.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0baed6b27..e5f18a75d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -553,6 +553,10 @@ from .lcp import (
 )
 from .learnr import LearnrIE
 from .lecture2go import Lecture2GoIE
+from .lecturio import (
+    LecturioIE,
+    LecturioCourseIE,
+)
 from .leeco import (
     LeIE,
     LePlaylistIE,
diff --git a/youtube_dl/extractor/lecturio.py b/youtube_dl/extractor/lecturio.py
new file mode 100644
index 000000000..62ff28e02
--- /dev/null
+++ b/youtube_dl/extractor/lecturio.py
@@ -0,0 +1,186 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    extract_attributes,
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    str_or_none,
+    url_or_none,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class LecturioBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://app.lecturio.com/en/login'
+    _NETRC_MACHINE = 'lecturio'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        # Sets some cookies
+        _, urlh = self._download_webpage_handle(
+            self._LOGIN_URL, None, 'Downloading login popup')
+
+        def is_logged(url_handle):
+            return self._LOGIN_URL not in compat_str(url_handle.geturl())
+
+        # Already logged in
+        if is_logged(urlh):
+            return
+
+        login_form = {
+            'signin[email]': username,
+            'signin[password]': password,
+            'signin[remember]': 'on',
+        }
+
+        response, urlh = self._download_webpage_handle(
+            self._LOGIN_URL, None, 'Logging in',
+            data=urlencode_postdata(login_form))
+
+        # Logged in successfully
+        if is_logged(urlh):
+            return
+
+        errors = self._html_search_regex(
+            r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
+            'errors', default=None)
+        if errors:
+            raise ExtractorError('Unable to login: %s' % errors, expected=True)
+        raise ExtractorError('Unable to log in')
+
+
+class LecturioIE(LecturioBaseIE):
+    _VALID_URL = r'https://app\.lecturio\.com/[^/]+/(?P<id>[^/?#&]+)\.lecture'
+    _TEST = {
+        'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
+        'md5': 'f576a797a5b7a5e4e4bbdfc25a6a6870',
+        'info_dict': {
+            'id': '39634',
+            'ext': 'mp4',
+            'title': 'Important Concepts and Terms – Introduction to Microbiology',
+        },
+        'skip': 'Requires lecturio account credentials',
+    }
+
+    _CC_LANGS = {
+        'German': 'de',
+        'English': 'en',
+        'Spanish': 'es',
+        'French': 'fr',
+        'Polish': 'pl',
+        'Russian': 'ru',
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://app.lecturio.com/en/lecture/%s/player.html' % display_id,
+            display_id)
+
+        lecture_id = self._search_regex(
+            r'lecture_id\s*=\s*(?:L_)?(\d+)', webpage, 'lecture id')
+
+        api_url = self._search_regex(
+            r'lectureDataLink\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'api url', group='url')
+
+        video = self._download_json(api_url, display_id)
+
+        title = video['title'].strip()
+
+        formats = []
+        for format_ in video['content']['media']:
+            if not isinstance(format_, dict):
+                continue
+            file_ = format_.get('file')
+            if not file_:
+                continue
+            ext = determine_ext(file_)
+            if ext == 'smil':
+                # smil contains only broken RTMP formats anyway
+                continue
+            file_url = url_or_none(file_)
+            if not file_url:
+                continue
+            label = str_or_none(format_.get('label'))
+            filesize = int_or_none(format_.get('fileSize'))
+            formats.append({
+                'url': file_url,
+                'format_id': label,
+                'filesize': float_or_none(filesize, invscale=1000)
+            })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        automatic_captions = {}
+        cc = self._parse_json(
+            self._search_regex(
+                r'subtitleUrls\s*:\s*({.+?})\s*,', webpage, 'subtitles',
+                default='{}'), display_id, fatal=False)
+        for cc_label, cc_url in cc.items():
+            cc_url = url_or_none(cc_url)
+            if not cc_url:
+                continue
+            sub_dict = automatic_captions if 'auto-translated' in cc_label else subtitles
+            lang = self._search_regex(
+                r'/([a-z]{2})_', cc_url, 'lang', default=cc_label.split()[0])
+            sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
+                'url': cc_url,
+            })
+
+        return {
+            'id': lecture_id,
+            'title': title,
+            'formats': formats,
+            'subtitles': subtitles,
+            'automatic_captions': automatic_captions,
+        }
+
+
+class LecturioCourseIE(LecturioBaseIE):
+    _VALID_URL = r'https://app\.lecturio\.com/[^/]+/(?P<id>[^/?#&]+)\.course'
+    _TEST = {
+        'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
+        'info_dict': {
+            'id': 'microbiology-introduction',
+            'title': 'Microbiology: Introduction',
+        },
+        'playlist_count': 45,
+        'skip': 'Requires lecturio account credentials',
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        entries = []
+        for mobj in re.finditer(
+                r'(?s)<[^>]+\bdata-url=(["\'])(?:(?!\1).)+\.lecture\b[^>]+>',
+                webpage):
+            params = extract_attributes(mobj.group(0))
+            lecture_url = urljoin(url, params.get('data-url'))
+            lecture_id = params.get('data-id')
+            entries.append(self.url_result(
+                lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
+
+        title = self._search_regex(
+            r'<span[^>]+class=["\']content-title[^>]+>([^<]+)', webpage,
+            'title', default=None)
+
+        return self.playlist_result(entries, display_id, title)

From ebb0449049c198f04103502c95a13171b854d1c7 Mon Sep 17 00:00:00 2001
From: ealgase <mostdigitsofpi@gmail.com>
Date: Thu, 6 Dec 2018 15:36:08 -0500
Subject: [PATCH 223/510] [xvideos] Switch to HTTPS (closes #18422)

---
 youtube_dl/extractor/xvideos.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index efee95651..ec2d913fc 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -45,7 +45,7 @@ class XVideosIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://www.xvideos.com/video%s/' % video_id, video_id)
+            'https://www.xvideos.com/video%s/' % video_id, video_id)
 
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:

From 8c5879715f4d979b83c49d44a9094307247097ba Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Thu, 6 Dec 2018 21:41:02 +0100
Subject: [PATCH 224/510] [ard:mediathek] Fix title and description extraction
 (closes #18349)

---
 youtube_dl/extractor/ard.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6bf8f61eb..84e96f769 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -173,13 +173,18 @@ class ARDMediathekIE(InfoExtractor):
         title = self._html_search_regex(
             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
              r'<meta name="dcterms\.title" content="(.*?)"/>',
-             r'<h4 class="headline">(.*?)</h4>'],
+             r'<h4 class="headline">(.*?)</h4>',
+             r'<title[^>]*>(.*?)</title>'],
             webpage, 'title')
         description = self._html_search_meta(
             'dcterms.abstract', webpage, 'description', default=None)
         if description is None:
             description = self._html_search_meta(
-                'description', webpage, 'meta description')
+                'description', webpage, 'meta description', default=None)
+        if description is None:
+            description = self._html_search_regex(
+                r'<p\s+class="teasertext">(.+?)</p>',
+                webpage, 'teaser text', default=None)
 
         # Thumbnail is sometimes not present.
         # It is in the mobile version, but that seems to use a different URL

From c3c098dcf2826ab4d668a92c9137cca2c0c42a4f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 7 Dec 2018 18:52:01 +0100
Subject: [PATCH 225/510] [hotstar] fix video data extraction(closes #18386)

---
 youtube_dl/extractor/hotstar.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 45aa5e7ea..8de9c4faf 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -79,7 +79,7 @@ class HotStarIE(HotStarBaseIE):
             r'<script>window\.APP_STATE\s*=\s*({.+?})</script>',
             webpage, 'app state'), video_id)
         video_data = {}
-        getters = (
+        getters = list(
             lambda x, k=k: x['initialState']['content%s' % k]['content']
             for k in ('Data', 'Detail')
         )
@@ -87,6 +87,7 @@ class HotStarIE(HotStarBaseIE):
             content = try_get(v, getters, dict)
             if content and content.get('contentId') == video_id:
                 video_data = content
+                break
 
         title = video_data['title']
 

From 9235b5091cedcc21c8dc32d4b292340edeee4ed0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Dec 2018 23:57:40 +0700
Subject: [PATCH 226/510] [iprima] Relax _VALID_URL (closes #18453)

---
 youtube_dl/extractor/iprima.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 1d58d6e85..11a6629d2 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:play|prima)\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://(?:play|prima|www)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_BYPASS = False
 
     _TESTS = [{
@@ -41,6 +41,9 @@ class IPrimaIE(InfoExtractor):
         # iframe prima.iprima.cz
         'url': 'https://prima.iprima.cz/porady/jak-se-stavi-sen/rodina-rathousova-praha',
         'only_matching': True,
+    }, {
+        'url': 'http://www.iprima.cz/filmy/desne-rande',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 1d88b3e6e6e59e4b52305faf6c1bf1fd69c555ee Mon Sep 17 00:00:00 2001
From: aegamesi <admin.aegamesi@gmail.com>
Date: Sat, 29 Apr 2017 22:56:33 -0700
Subject: [PATCH 227/510] [YoutubeDL] Recognize expires=0 as session cookies
 and send session cookies with requests

---
 youtube_dl/YoutubeDL.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 38ba43a97..2433f74f4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2300,7 +2300,13 @@ class YoutubeDL(object):
             self.cookiejar = compat_cookiejar.MozillaCookieJar(
                 opts_cookiefile)
             if os.access(opts_cookiefile, os.R_OK):
-                self.cookiejar.load()
+                self.cookiejar.load(ignore_discard=True, ignore_expires=True)
+                # Force CookieJar to treat 'expires=0' cookies as session/discard cookies
+                # Fixes https://bugs.python.org/issue17164
+                for cookie in self.cookiejar:
+                    if cookie.expires == 0:
+                        cookie.expires = None
+                        cookie.discard = True
 
         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
         if opts_proxy is not None:

From 1bab3437046646da4ebe2b8e0c7fdc25aa1072ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Dec 2018 06:00:32 +0700
Subject: [PATCH 228/510] [YoutubeDL] Introduce YoutubeDLCookieJar and clarify
 the rationale behind session cookies (closes #12929)

---
 youtube_dl/YoutubeDL.py | 12 +++---------
 youtube_dl/utils.py     | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2433f74f4..4493fd0e1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -88,6 +88,7 @@ from .utils import (
     version_tuple,
     write_json_file,
     write_string,
+    YoutubeDLCookieJar,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
 )
@@ -558,7 +559,7 @@ class YoutubeDL(object):
         self.restore_console_title()
 
         if self.params.get('cookiefile') is not None:
-            self.cookiejar.save()
+            self.cookiejar.save(ignore_discard=True, ignore_expires=True)
 
     def trouble(self, message=None, tb=None):
         """Determine action to take when a download problem appears.
@@ -2297,16 +2298,9 @@ class YoutubeDL(object):
             self.cookiejar = compat_cookiejar.CookieJar()
         else:
             opts_cookiefile = expand_path(opts_cookiefile)
-            self.cookiejar = compat_cookiejar.MozillaCookieJar(
-                opts_cookiefile)
+            self.cookiejar = YoutubeDLCookieJar(opts_cookiefile)
             if os.access(opts_cookiefile, os.R_OK):
                 self.cookiejar.load(ignore_discard=True, ignore_expires=True)
-                # Force CookieJar to treat 'expires=0' cookies as session/discard cookies
-                # Fixes https://bugs.python.org/issue17164
-                for cookie in self.cookiejar:
-                    if cookie.expires == 0:
-                        cookie.expires = None
-                        cookie.discard = True
 
         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
         if opts_proxy is not None:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0b1c7cd6c..62e769fd5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -39,6 +39,7 @@ from .compat import (
     compat_HTMLParser,
     compat_basestring,
     compat_chr,
+    compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
     compat_etree_fromstring,
     compat_expanduser,
@@ -1139,6 +1140,33 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             req, **kwargs)
 
 
+class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
+    def save(self, filename=None, ignore_discard=False, ignore_expires=False):
+        # Store session cookies with `expires` set to 0 instead of an empty
+        # string
+        for cookie in self:
+            if cookie.expires is None:
+                cookie.expires = 0
+        compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
+
+    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
+        compat_cookiejar.MozillaCookieJar.load(self, filename, ignore_discard, ignore_expires)
+        # Session cookies are denoted by either `expires` field set to
+        # an empty string or 0. MozillaCookieJar only recognizes the former
+        # (see [1]). So we need force the latter to be recognized as session
+        # cookies on our own.
+        # Session cookies may be important for cookies-based authentication,
+        # e.g. usually, when user does not check 'Remember me' check box while
+        # logging in on a site, some important cookies are stored as session
+        # cookies so that not recognizing them will result in failed login.
+        # 1. https://bugs.python.org/issue17164
+        for cookie in self:
+            # Treat `expires=0` cookies as session cookies
+            if cookie.expires == 0:
+                cookie.expires = None
+                cookie.discard = True
+
+
 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
     def __init__(self, cookiejar=None):
         compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)

From 5f47a60c5d66d65b505131a672a3bad67ddaa00f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 9 Dec 2018 09:35:17 +0100
Subject: [PATCH 229/510] [imgur] improve gallery and album detection and
 extraction(closes #9133)(closes #16577)(closes #17223)(closes #18404)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/imgur.py      | 86 +++++++++++++++---------------
 2 files changed, 44 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e5f18a75d..e28db6968 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -483,6 +483,7 @@ from .imdb import (
 from .imgur import (
     ImgurIE,
     ImgurAlbumIE,
+    ImgurGalleryIE,
 )
 from .ina import InaIE
 from .inc import IncIE
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index ecc958a17..0eb54db3f 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:gallery|(?:topic|r)/[^/]+)/)?(?P<id>[a-zA-Z0-9]{6,})(?:[/?#&]+|\.[a-z0-9]+)?$'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|(?:t(?:opic)?|r)/[^/]+)/)(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -20,28 +20,9 @@ class ImgurIE(InfoExtractor):
             'id': 'A61SaA1',
             'ext': 'mp4',
             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
-            'description': 'Imgur: The magic of the Internet',
         },
     }, {
         'url': 'https://imgur.com/A61SaA1',
-        'info_dict': {
-            'id': 'A61SaA1',
-            'ext': 'mp4',
-            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
-            'description': 'Imgur: The magic of the Internet',
-        },
-    }, {
-        'url': 'https://imgur.com/gallery/YcAQlkx',
-        'info_dict': {
-            'id': 'YcAQlkx',
-            'ext': 'mp4',
-            'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
-        }
-    }, {
-        'url': 'http://imgur.com/topic/Funny/N8rOudd',
-        'only_matching': True,
-    }, {
-        'url': 'http://imgur.com/r/aww/VQcQPhM',
         'only_matching': True,
     }, {
         'url': 'https://i.imgur.com/crGpqCV.mp4',
@@ -50,8 +31,8 @@ class ImgurIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        gifv_url = 'https://i.imgur.com/{id}.gifv'.format(id=video_id)
-        webpage = self._download_webpage(gifv_url, video_id)
+        webpage = self._download_webpage(
+            'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id)
 
         width = int_or_none(self._og_search_property(
             'video:width', webpage, default=None))
@@ -72,7 +53,6 @@ class ImgurIE(InfoExtractor):
                 'format_id': m.group('type').partition('/')[2],
                 'url': self._proto_relative_url(m.group('src')),
                 'ext': mimetype2ext(m.group('type')),
-                'acodec': 'none',
                 'width': width,
                 'height': height,
                 'http_headers': {
@@ -107,44 +87,64 @@ class ImgurIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'description': self._og_search_description(webpage, default=None),
             'title': self._og_search_title(webpage),
         }
 
 
-class ImgurAlbumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:a|gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{5})(?:[/?#&]+)?$'
+class ImgurGalleryIE(InfoExtractor):
+    IE_NAME = 'imgur:gallery'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'http://imgur.com/gallery/Q95ko',
         'info_dict': {
             'id': 'Q95ko',
+            'title': 'Adding faces make every GIF better',
         },
         'playlist_count': 25,
     }, {
-        'url': 'http://imgur.com/a/j6Orj',
+        'url': 'http://imgur.com/topic/Aww/ll5Vk',
         'only_matching': True,
     }, {
-        'url': 'http://imgur.com/topic/Aww/ll5Vk',
+        'url': 'https://imgur.com/gallery/YcAQlkx',
+        'info_dict': {
+            'id': 'YcAQlkx',
+            'ext': 'mp4',
+            'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
+        }
+    }, {
+        'url': 'http://imgur.com/topic/Funny/N8rOudd',
+        'only_matching': True,
+    }, {
+        'url': 'http://imgur.com/r/aww/VQcQPhM',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        album_id = self._match_id(url)
+        gallery_id = self._match_id(url)
 
-        album_images = self._download_json(
-            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
-            album_id, fatal=False)
+        data = self._download_json(
+            'https://imgur.com/gallery/%s.json' % gallery_id,
+            gallery_id)['data']['image']
 
-        if album_images:
-            data = album_images.get('data')
-            if data and isinstance(data, dict):
-                images = data.get('images')
-                if images and isinstance(images, list):
-                    entries = [
-                        self.url_result('http://imgur.com/%s' % image['hash'])
-                        for image in images if image.get('hash')]
-                    return self.playlist_result(entries, album_id)
+        if data.get('is_album'):
+            entries = [
+                self.url_result('http://imgur.com/%s' % image['hash'], ImgurIE.ie_key(), image['hash'])
+                for image in data['album_images']['images'] if image.get('hash')]
+            return self.playlist_result(entries, gallery_id, data.get('title'), data.get('description'))
 
-        # Fallback to single video
-        return self.url_result('http://imgur.com/%s' % album_id, ImgurIE.ie_key())
+        return self.url_result('http://imgur.com/%s' % gallery_id, ImgurIE.ie_key(), gallery_id)
+
+
+class ImgurAlbumIE(ImgurGalleryIE):
+    IE_NAME = 'imgur:album'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P<id>[a-zA-Z0-9]+)'
+
+    _TESTS = [{
+        'url': 'http://imgur.com/a/j6Orj',
+        'info_dict': {
+            'id': 'j6Orj',
+            'title': 'A Literary Analysis of "Star Wars: The Force Awakens"',
+        },
+        'playlist_count': 12,
+    }]

From 3ad6dabd33307c0125fd462c4988083e360c40ad Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 9 Dec 2018 10:04:00 +0100
Subject: [PATCH 230/510] [aenetworks] add support for History Vault(closes
 #18460)

---
 youtube_dl/extractor/aenetworks.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 398e56ea3..85ec6392d 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -22,18 +22,19 @@ class AENetworksBaseIE(ThePlatformIE):
 
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
-    IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
+    IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?
                         (?P<domain>
-                            (?:history|aetv|mylifetime|lifetimemovieclub)\.com|
+                            (?:history(?:vault)?|aetv|mylifetime|lifetimemovieclub)\.com|
                             fyi\.tv
                         )/
                         (?:
                             shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|
                             movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?|
-                            specials/(?P<special_display_id>[^/]+)/full-special
+                            specials/(?P<special_display_id>[^/]+)/full-special|
+                            collections/[^/]+/(?P<collection_display_id>[^/]+)
                         )
                     '''
     _TESTS = [{
@@ -80,6 +81,9 @@ class AENetworksIE(AENetworksBaseIE):
     }, {
         'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special',
         'only_matching': True
+    }, {
+        'url': 'https://www.historyvault.com/collections/america-the-story-of-us/westward',
+        'only_matching': True
     }]
     _DOMAIN_TO_REQUESTOR_ID = {
         'history.com': 'HISTORY',
@@ -90,9 +94,9 @@ class AENetworksIE(AENetworksBaseIE):
     }
 
     def _real_extract(self, url):
-        domain, show_path, movie_display_id, special_display_id = re.match(self._VALID_URL, url).groups()
-        display_id = show_path or movie_display_id or special_display_id
-        webpage = self._download_webpage(url, display_id)
+        domain, show_path, movie_display_id, special_display_id, collection_display_id = re.match(self._VALID_URL, url).groups()
+        display_id = show_path or movie_display_id or special_display_id or collection_display_id
+        webpage = self._download_webpage(url, display_id, headers=self.geo_verification_headers())
         if show_path:
             url_parts = show_path.split('/')
             url_parts_len = len(url_parts)

From 5ee7ae5c7577c9c137a7b38edd5ad01ae3a40376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Dec 2018 22:28:24 +0700
Subject: [PATCH 231/510] [teachable] Add support for teachable based platform
 sites (closes #5451, closes #18150, closes #18272)

---
 youtube_dl/extractor/extractors.py            |   8 +-
 youtube_dl/extractor/generic.py               |   5 +
 .../extractor/{upskill.py => teachable.py}    | 129 ++++++++++++++----
 3 files changed, 115 insertions(+), 27 deletions(-)
 rename youtube_dl/extractor/{upskill.py => teachable.py} (52%)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e28db6968..6a5d12ab1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1091,6 +1091,10 @@ from .tass import TassIE
 from .tastytrade import TastyTradeIE
 from .tbs import TBSIE
 from .tdslifeway import TDSLifewayIE
+from .teachable import (
+    TeachableIE,
+    TeachableCourseIE,
+)
 from .teachertube import (
     TeacherTubeIE,
     TeacherTubeUserIE,
@@ -1240,10 +1244,6 @@ from .uplynk import (
     UplynkIE,
     UplynkPreplayIE,
 )
-from .upskill import (
-    UpskillIE,
-    UpskillCourseIE,
-)
 from .urort import UrortIE
 from .urplay import URPlayIE
 from .usanetwork import USANetworkIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 59cf03faf..65b482333 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -109,6 +109,7 @@ from .vice import ViceIE
 from .xfileshare import XFileShareIE
 from .cloudflarestream import CloudflareStreamIE
 from .peertube import PeerTubeIE
+from .teachable import TeachableIE
 from .indavideo import IndavideoEmbedIE
 from .apa import APAIE
 from .foxnews import FoxNewsIE
@@ -3112,6 +3113,10 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 peertube_urls, video_id, video_title, ie=PeerTubeIE.ie_key())
 
+        teachable_url = TeachableIE._extract_url(webpage, url)
+        if teachable_url:
+            return self.url_result(teachable_url)
+
         indavideo_urls = IndavideoEmbedIE._extract_urls(webpage)
         if indavideo_urls:
             return self.playlist_from_matches(
diff --git a/youtube_dl/extractor/upskill.py b/youtube_dl/extractor/teachable.py
similarity index 52%
rename from youtube_dl/extractor/upskill.py
rename to youtube_dl/extractor/teachable.py
index 30297b4dd..a0c46b2e6 100644
--- a/youtube_dl/extractor/upskill.py
+++ b/youtube_dl/extractor/teachable.py
@@ -14,20 +14,38 @@ from ..utils import (
 )
 
 
-class UpskillBaseIE(InfoExtractor):
-    _LOGIN_URL = 'http://upskillcourses.com/sign_in'
-    _NETRC_MACHINE = 'upskill'
+class TeachableBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'teachable'
+    _URL_PREFIX = 'teachable:'
+
+    _SITES = {
+        # Only notable ones here
+        'upskillcourses.com': 'upskill',
+        'academy.gns3.com': 'gns3',
+        'academyhacker.com': 'academyhacker',
+        'stackskills.com': 'stackskills',
+        'market.saleshacker.com': 'saleshacker',
+        'learnability.org': 'learnability',
+        'edurila.com': 'edurila',
+    }
+
+    _VALID_URL_SUB_TUPLE = (_URL_PREFIX, '|'.join(re.escape(site) for site in _SITES.keys()))
 
     def _real_initialize(self):
-        self._login()
+        self._logged_in = False
 
-    def _login(self):
-        username, password = self._get_login_info()
+    def _login(self, site):
+        if self._logged_in:
+            return
+
+        username, password = self._get_login_info(
+            netrc_machine=self._SITES.get(site, site))
         if username is None:
             return
 
         login_page, urlh = self._download_webpage_handle(
-            self._LOGIN_URL, None, 'Downloading login page')
+            'https://%s/sign_in' % site, None,
+            'Downloading %s login page' % site)
 
         login_url = compat_str(urlh.geturl())
 
@@ -46,18 +64,24 @@ class UpskillBaseIE(InfoExtractor):
             post_url = urljoin(login_url, post_url)
 
         response = self._download_webpage(
-            post_url, None, 'Logging in',
+            post_url, None, 'Logging in to %s' % site,
             data=urlencode_postdata(login_form),
             headers={
                 'Content-Type': 'application/x-www-form-urlencoded',
                 'Referer': login_url,
             })
 
+        if '>I accept the new Privacy Policy<' in response:
+            raise ExtractorError(
+                'Unable to login: %s asks you to accept new Privacy Policy. '
+                'Go to https://%s/ and accept.' % (site, site), expected=True)
+
         # Successful login
         if any(re.search(p, response) for p in (
                 r'class=["\']user-signout',
                 r'<a[^>]+\bhref=["\']/sign_out',
                 r'>\s*Log out\s*<')):
+            self._logged_in = True
             return
 
         message = get_element_by_class('alert', response)
@@ -68,8 +92,14 @@ class UpskillBaseIE(InfoExtractor):
         raise ExtractorError('Unable to log in')
 
 
-class UpskillIE(UpskillBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?upskillcourses\.com/courses/[^/]+/lectures/(?P<id>\d+)'
+class TeachableIE(TeachableBaseIE):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        %shttps?://(?P<site_t>[^/]+)|
+                        https?://(?:www\.)?(?P<site>%s)
+                    )
+                    /courses/[^/]+/lectures/(?P<id>\d+)
+                    ''' % TeachableBaseIE._VALID_URL_SUB_TUPLE
 
     _TESTS = [{
         'url': 'http://upskillcourses.com/courses/essential-web-developer-course/lectures/1747100',
@@ -77,7 +107,7 @@ class UpskillIE(UpskillBaseIE):
             'id': 'uzw6zw58or',
             'ext': 'mp4',
             'title': 'Welcome to the Course!',
-            'description': 'md5:8d66c13403783370af62ca97a7357bdd',
+            'description': 'md5:65edb0affa582974de4625b9cdea1107',
             'duration': 138.763,
             'timestamp': 1479846621,
             'upload_date': '20161122',
@@ -88,10 +118,38 @@ class UpskillIE(UpskillBaseIE):
     }, {
         'url': 'http://upskillcourses.com/courses/119763/lectures/1747100',
         'only_matching': True,
+    }, {
+        'url': 'https://academy.gns3.com/courses/423415/lectures/6885939',
+        'only_matching': True,
+    }, {
+        'url': 'teachable:https://upskillcourses.com/courses/essential-web-developer-course/lectures/1747100',
+        'only_matching': True,
     }]
 
+    @staticmethod
+    def _is_teachable(webpage):
+        return 'teachableTracker.linker:autoLink' in webpage and re.search(
+            r'<link[^>]+href=["\']https?://process\.fs\.teachablecdn\.com',
+            webpage)
+
+    @staticmethod
+    def _extract_url(webpage, source_url):
+        if not TeachableIE._is_teachable(webpage):
+            print('NOT TEACHABLE')
+            return
+        if re.match(r'https?://[^/]+/(?:courses|p)', source_url):
+            return '%s%s' % (TeachableBaseIE._URL_PREFIX, source_url)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        site = mobj.group('site') or mobj.group('site_t')
+        video_id = mobj.group('id')
+
+        self._login(site)
+
+        prefixed = url.startswith(self._URL_PREFIX)
+        if prefixed:
+            url = url[len(self._URL_PREFIX):]
 
         webpage = self._download_webpage(url, video_id)
 
@@ -113,12 +171,18 @@ class UpskillIE(UpskillBaseIE):
         }
 
 
-class UpskillCourseIE(UpskillBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?upskillcourses\.com/courses/(?:enrolled/)?(?P<id>[^/?#&]+)'
+class TeachableCourseIE(TeachableBaseIE):
+    _VALID_URL = r'''(?x)
+                        (?:
+                            %shttps?://(?P<site_t>[^/]+)|
+                            https?://(?:www\.)?(?P<site>%s)
+                        )
+                        /(?:courses|p)/(?:enrolled/)?(?P<id>[^/?#&]+)
+                    ''' % TeachableBaseIE._VALID_URL_SUB_TUPLE
     _TESTS = [{
         'url': 'http://upskillcourses.com/courses/essential-web-developer-course/',
         'info_dict': {
-            'id': '119763',
+            'id': 'essential-web-developer-course',
             'title': 'The Essential Web Developer Course (Free)',
         },
         'playlist_count': 192,
@@ -128,21 +192,37 @@ class UpskillCourseIE(UpskillBaseIE):
     }, {
         'url': 'http://upskillcourses.com/courses/enrolled/119763',
         'only_matching': True,
+    }, {
+        'url': 'https://academy.gns3.com/courses/enrolled/423415',
+        'only_matching': True,
+    }, {
+        'url': 'teachable:https://learn.vrdev.school/p/gear-vr-developer-mini',
+        'only_matching': True,
+    }, {
+        'url': 'teachable:https://filmsimplified.com/p/davinci-resolve-15-crash-course',
+        'only_matching': True,
     }]
 
     @classmethod
     def suitable(cls, url):
-        return False if UpskillIE.suitable(url) else super(
-            UpskillCourseIE, cls).suitable(url)
+        return False if TeachableIE.suitable(url) else super(
+            TeachableCourseIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        course_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        site = mobj.group('site') or mobj.group('site_t')
+        course_id = mobj.group('id')
+
+        self._login(site)
+
+        prefixed = url.startswith(self._URL_PREFIX)
+        if prefixed:
+            prefix = self._URL_PREFIX
+            url = url[len(prefix):]
 
         webpage = self._download_webpage(url, course_id)
 
-        course_id = self._search_regex(
-            r'data-course-id=["\'](\d+)', webpage, 'course id',
-            default=course_id)
+        url_base = 'https://%s/' % site
 
         entries = []
 
@@ -162,10 +242,13 @@ class UpskillCourseIE(UpskillBaseIE):
             title = self._html_search_regex(
                 r'<span[^>]+class=["\']lecture-name[^>]+>([^<]+)', li,
                 'title', default=None)
+            entry_url = urljoin(url_base, lecture_url)
+            if prefixed:
+                entry_url = self._URL_PREFIX + entry_url
             entries.append(
                 self.url_result(
-                    urljoin('http://upskillcourses.com/', lecture_url),
-                    ie=UpskillIE.ie_key(), video_id=lecture_id,
+                    entry_url,
+                    ie=TeachableIE.ie_key(), video_id=lecture_id,
                     video_title=clean_html(title)))
 
         course_title = self._html_search_regex(

From 9e02c2c704d15d56b92ebe2fc8d481f99f5d106d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Dec 2018 22:56:33 +0700
Subject: [PATCH 232/510] [YoutubeDLCookieJar] Add test for keeping session
 cookies

---
 test/test_YoutubeDLCookieJar.py           | 34 +++++++++++++++++++++++
 test/testdata/cookies/session_cookies.txt |  6 ++++
 2 files changed, 40 insertions(+)
 create mode 100644 test/test_YoutubeDLCookieJar.py
 create mode 100644 test/testdata/cookies/session_cookies.txt

diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py
new file mode 100644
index 000000000..6a8243590
--- /dev/null
+++ b/test/test_YoutubeDLCookieJar.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import tempfile
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.utils import YoutubeDLCookieJar
+
+
+class TestYoutubeDLCookieJar(unittest.TestCase):
+    def test_keep_session_cookies(self):
+        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
+        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        tf = tempfile.NamedTemporaryFile(delete=False)
+        try:
+            cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True)
+            temp = tf.read().decode('utf-8')
+            self.assertTrue(re.search(
+                r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp))
+            self.assertTrue(re.search(
+                r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpires0\s+YoutubeDLExpires0Value', temp))
+        finally:
+            tf.close()
+            os.remove(tf.name)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/testdata/cookies/session_cookies.txt b/test/testdata/cookies/session_cookies.txt
new file mode 100644
index 000000000..91e5c9231
--- /dev/null
+++ b/test/testdata/cookies/session_cookies.txt
@@ -0,0 +1,6 @@
+# Netscape HTTP Cookie File
+# http://curl.haxx.se/rfc/cookie_spec.html
+# This is a generated file!  Do not edit.
+
+www.foobar.foobar	FALSE	/	TRUE	0	YoutubeDLExpires0	YoutubeDLExpires0Value
+www.foobar.foobar	FALSE	/	TRUE	0	YoutubeDLExpiresEmpty	YoutubeDLExpiresEmptyValue

From 24cc64254c03c56229654b13d833c600d7048bbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Dec 2018 23:08:16 +0700
Subject: [PATCH 233/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 689d07826..6afee3a28 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Keep session cookies in cookie file between runs
+* [YoutubeDL] Recognize session cookies with expired set to 0 (#12929)
+
+Extractors
++ [teachable] Add support for teachable platform sites (#5451, #18150, #18272)
++ [aenetworks] Add support for historyvault.com (#18460)
+* [imgur] Improve gallery and album detection and extraction (#9133, #16577,
+  #17223, #18404)
+* [iprima] Relax URL regular expression (#18453)
+* [hotstar] Fix video data extraction (#18386)
+* [ard:mediathek] Fix title and description extraction (#18349, #18371)
+* [xvideos] Switch to HTTPS (#18422, #18427)
++ [lecturio] Add support for lecturio.com (#18405)
++ [nrktv:series] Add support for extra materials
+* [nrktv:season,series] Fix extraction (#17159, #17258)
+* [nrktv] Relax URL regular expression (#18304, #18387)
+* [yourporn] Fix extraction (#18424, #18425)
+* [tbs] Fix info extraction (#18403)
++ [gamespot] Add support for review URLs
+
+
 version 2018.12.03
 
 Core

From cefe42c412168e95b78dcefc5cdba608dba7dd02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Dec 2018 23:11:32 +0700
Subject: [PATCH 234/510] release 2018.12.09

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 11 +++++++----
 youtube_dl/version.py     |  2 +-
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4b35244a8..1ccf410a7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.03*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.09*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.12.03
+[debug] youtube-dl version 2018.12.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 6afee3a28..9aef67b52 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.12.09
 
 Core
 * [YoutubeDL] Keep session cookies in cookie file between runs
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 837b0199b..31d20f255 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -33,7 +33,7 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
- - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
+ - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault
  - **afreecatv**: afreecatv.com
  - **AirMozilla**
  - **AliExpressLive**
@@ -376,7 +376,8 @@
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
  - **Imgur**
- - **ImgurAlbum**
+ - **imgur:album**
+ - **imgur:gallery**
  - **Ina**
  - **Inc**
  - **IndavideoEmbed**
@@ -435,6 +436,8 @@
  - **Le**: 乐视网
  - **Learnr**
  - **Lecture2Go**
+ - **Lecturio**
+ - **LecturioCourse**
  - **LEGO**
  - **Lemonde**
  - **Lenta**
@@ -853,6 +856,8 @@
  - **TastyTrade**
  - **TBS**
  - **TDSLifeway**
+ - **Teachable**
+ - **TeachableCourse**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
  - **TeachingChannel**
@@ -961,8 +966,6 @@
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
- - **Upskill**
- - **UpskillCourse**
  - **Urort**: NRK P3 Urørt
  - **URPlay**
  - **USANetwork**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8e1203892..d15b9583f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.12.03'
+__version__ = '2018.12.09'

From 59c3940165efaee6705ff06ef4fc4ac2c701107d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Dec 2018 01:37:10 +0700
Subject: [PATCH 235/510] [ard:mediathek] Add support for
 classic.ardmediathek.de (closes #18473)

---
 youtube_dl/extractor/ard.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 84e96f769..2e1536a1b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -21,7 +21,7 @@ from ..compat import compat_etree_fromstring
 
 class ARDMediathekIE(InfoExtractor):
     IE_NAME = 'ARD:mediathek'
-    _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
+    _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
 
     _TESTS = [{
         # available till 26.07.2022
@@ -51,6 +51,9 @@ class ARDMediathekIE(InfoExtractor):
         # audio
         'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
         'only_matching': True,
+    }, {
+        'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
+        'only_matching': True,
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):

From 6e29458f2479c4535f8605032b9492f52bb37f17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Dec 2018 04:30:00 +0700
Subject: [PATCH 236/510] [test/testdata/cookies/session_cookies.txt] Fix empty
 expires test data

---
 test/testdata/cookies/session_cookies.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/testdata/cookies/session_cookies.txt b/test/testdata/cookies/session_cookies.txt
index 91e5c9231..f6996f031 100644
--- a/test/testdata/cookies/session_cookies.txt
+++ b/test/testdata/cookies/session_cookies.txt
@@ -2,5 +2,5 @@
 # http://curl.haxx.se/rfc/cookie_spec.html
 # This is a generated file!  Do not edit.
 
+www.foobar.foobar	FALSE	/	TRUE		YoutubeDLExpiresEmpty	YoutubeDLExpiresEmptyValue
 www.foobar.foobar	FALSE	/	TRUE	0	YoutubeDLExpires0	YoutubeDLExpires0Value
-www.foobar.foobar	FALSE	/	TRUE	0	YoutubeDLExpiresEmpty	YoutubeDLExpiresEmptyValue

From 102a4e54c5c0819233773cc15398bc901a218f4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Dec 2018 10:10:28 +0700
Subject: [PATCH 237/510] [teachable] Remove debug output

---
 youtube_dl/extractor/teachable.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/teachable.py b/youtube_dl/extractor/teachable.py
index a0c46b2e6..47ac95ee8 100644
--- a/youtube_dl/extractor/teachable.py
+++ b/youtube_dl/extractor/teachable.py
@@ -135,7 +135,6 @@ class TeachableIE(TeachableBaseIE):
     @staticmethod
     def _extract_url(webpage, source_url):
         if not TeachableIE._is_teachable(webpage):
-            print('NOT TEACHABLE')
             return
         if re.match(r'https?://[^/]+/(?:courses|p)', source_url):
             return '%s%s' % (TeachableBaseIE._URL_PREFIX, source_url)

From 13e17cd28e7f9e3bd8be4fa7b073a5cb96f5959f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 10 Dec 2018 14:59:57 +0100
Subject: [PATCH 238/510] [uol] fix format url extraction(closes 18480)

---
 youtube_dl/extractor/uol.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/uol.py b/youtube_dl/extractor/uol.py
index e67083004..08f0c072e 100644
--- a/youtube_dl/extractor/uol.py
+++ b/youtube_dl/extractor/uol.py
@@ -61,7 +61,7 @@ class UOLIE(InfoExtractor):
             'height': 360,
         },
         '5': {
-            'width': 1080,
+            'width': 1280,
             'height': 720,
         },
         '6': {
@@ -80,6 +80,10 @@ class UOLIE(InfoExtractor):
             'width': 568,
             'height': 320,
         },
+        '11': {
+            'width': 640,
+            'height': 360,
+        }
     }
 
     def _real_extract(self, url):
@@ -111,19 +115,31 @@ class UOLIE(InfoExtractor):
             'ver': video_data.get('numRevision', 2),
             'r': 'http://mais.uol.com.br',
         }
+        for k in ('token', 'sign'):
+            v = video_data.get(k)
+            if v:
+                query[k] = v
+
         formats = []
         for f in video_data.get('formats', []):
             f_url = f.get('url') or f.get('secureUrl')
             if not f_url:
                 continue
+            f_url = update_url_query(f_url, query)
             format_id = str_or_none(f.get('id'))
+            if format_id == '10':
+                formats.extend(self._extract_m3u8_formats(
+                    f_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue
             fmt = {
                 'format_id': format_id,
-                'url': update_url_query(f_url, query),
+                'url': f_url,
+                'source_preference': 1,
             }
             fmt.update(self._FORMATS.get(format_id, {}))
             formats.append(fmt)
-        self._sort_formats(formats)
+        self._sort_formats(formats, ('height', 'width', 'source_preference', 'tbr', 'ext'))
 
         tags = []
         for tag in video_data.get('tags', []):

From 0a05cfabb6338be750474a7286ce0d72a4d7c142 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Dec 2018 23:45:02 +0700
Subject: [PATCH 239/510] [lecturio] Improve subtitles extraction (closes
 #18488)

---
 youtube_dl/extractor/lecturio.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecturio.py b/youtube_dl/extractor/lecturio.py
index 62ff28e02..0f1265cdf 100644
--- a/youtube_dl/extractor/lecturio.py
+++ b/youtube_dl/extractor/lecturio.py
@@ -136,9 +136,15 @@ class LecturioIE(LecturioBaseIE):
             cc_url = url_or_none(cc_url)
             if not cc_url:
                 continue
-            sub_dict = automatic_captions if 'auto-translated' in cc_label else subtitles
             lang = self._search_regex(
-                r'/([a-z]{2})_', cc_url, 'lang', default=cc_label.split()[0])
+                r'/([a-z]{2})_', cc_url, 'lang',
+                default=cc_label.split()[0] if cc_label else 'en')
+            original_lang = self._search_regex(
+                r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
+                default=None)
+            sub_dict = (automatic_captions
+                        if 'auto-translated' in cc_label or original_lang
+                        else subtitles)
             sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
                 'url': cc_url,
             })

From 8fe104947d6b772ccacd0b8af51d1adb16865e99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Dec 2018 22:25:12 +0700
Subject: [PATCH 240/510] [youtube] Fix multifeed extraction (closes #18531)

---
 youtube_dl/extractor/youtube.py | 52 ++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3f49f3889..c582ab2ff 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1712,30 +1712,36 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
+        if not smuggled_data.get('force_singlefeed', False):
             if not self._downloader.params.get('noplaylist'):
-                entries = []
-                feed_ids = []
-                multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
-                for feed in multifeed_metadata_list.split(','):
-                    # Unquote should take place before split on comma (,) since textual
-                    # fields may contain comma as well (see
-                    # https://github.com/rg3/youtube-dl/issues/8536)
-                    feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
-                    entries.append({
-                        '_type': 'url_transparent',
-                        'ie_key': 'Youtube',
-                        'url': smuggle_url(
-                            '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
-                            {'force_singlefeed': True}),
-                        'title': '%s (%s)' % (video_title, feed_data['title'][0]),
-                    })
-                    feed_ids.append(feed_data['id'][0])
-                self.to_screen(
-                    'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
-                    % (', '.join(feed_ids), video_id))
-                return self.playlist_result(entries, video_id, video_title, video_description)
-            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
+                multifeed_metadata_list = try_get(
+                    player_response,
+                    lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
+                    compat_str) or try_get(
+                    video_info, lambda x: x['multifeed_metadata_list'][0], compat_str)
+                if multifeed_metadata_list:
+                    entries = []
+                    feed_ids = []
+                    for feed in multifeed_metadata_list.split(','):
+                        # Unquote should take place before split on comma (,) since textual
+                        # fields may contain comma as well (see
+                        # https://github.com/rg3/youtube-dl/issues/8536)
+                        feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
+                        entries.append({
+                            '_type': 'url_transparent',
+                            'ie_key': 'Youtube',
+                            'url': smuggle_url(
+                                '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                                {'force_singlefeed': True}),
+                            'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                        })
+                        feed_ids.append(feed_data['id'][0])
+                    self.to_screen(
+                        'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
+                        % (', '.join(feed_ids), video_id))
+                    return self.playlist_result(entries, video_id, video_title, video_description)
+            else:
+                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
         if view_count is None:
             view_count = extract_view_count(video_info)

From 7f41a598b3fba1bcab2817de64a08941200aa3c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Dec 2018 23:08:14 +0700
Subject: [PATCH 241/510] [safari] Add support for learning.oreilly.com (closes
 #18510)

---
 youtube_dl/extractor/safari.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 30e2a38b4..c0d32a1b9 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -15,10 +15,10 @@ from ..utils import (
 
 
 class SafariBaseIE(InfoExtractor):
-    _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
+    _LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
     _NETRC_MACHINE = 'safari'
 
-    _API_BASE = 'https://www.safaribooksonline.com/api/v1'
+    _API_BASE = 'https://learning.oreilly.com/api/v1'
     _API_FORMAT = 'json'
 
     LOGGED_IN = False
@@ -76,7 +76,7 @@ class SafariIE(SafariBaseIE):
     IE_DESC = 'safaribooksonline.com online video'
     _VALID_URL = r'''(?x)
                         https?://
-                            (?:www\.)?safaribooksonline\.com/
+                            (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/
                             (?:
                                 library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
                                 videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
@@ -104,6 +104,9 @@ class SafariIE(SafariBaseIE):
     }, {
         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
         'only_matching': True,
+    }, {
+        'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
+        'only_matching': True,
     }]
 
     _PARTNER_ID = '1926081'
@@ -160,7 +163,7 @@ class SafariIE(SafariBaseIE):
 
 class SafariApiIE(SafariBaseIE):
     IE_NAME = 'safari:api'
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
@@ -185,7 +188,7 @@ class SafariCourseIE(SafariBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?safaribooksonline\.com/
+                            (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/
                             (?:
                                 library/view/[^/]+|
                                 api/v1/book|
@@ -213,6 +216,9 @@ class SafariCourseIE(SafariBaseIE):
     }, {
         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
         'only_matching': True,
+    }, {
+        'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
+        'only_matching': True,
     }]
 
     @classmethod

From c984196cf1c9eb34725091e07a4f094b7eea1d4f Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 15 Dec 2018 23:59:17 +0700
Subject: [PATCH 242/510] [README.md] Bind info dict URLs to a fixed blob
 (closes #18492)

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 35c3de512..a1d2904c0 100644
--- a/README.md
+++ b/README.md
@@ -1024,7 +1024,7 @@ After you have ensured this site is distributing its content legally, you can fo
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
 9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
@@ -1045,7 +1045,7 @@ Extractors are very fragile by nature since they depend on the layout of the sou
 
 ### Mandatory and optional metafields
 
-For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L75-L257) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
 
  - `id` (media identifier)
  - `title` (media title)
@@ -1053,7 +1053,7 @@ For extraction to work youtube-dl relies on metadata your extractor extracts and
 
 In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` as mandatory. Thus the aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken.
 
-[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+[Any field](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
 #### Example
 

From 21c340b83fb41094ef59b87d52c4eb1d90d1df04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Dec 2018 19:35:48 +0700
Subject: [PATCH 243/510] [youtube] Fix mark watched (closes #18546)

---
 youtube_dl/extractor/youtube.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c582ab2ff..44c25c11c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -48,6 +48,7 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     uppercase_escape,
+    url_or_none,
     urlencode_postdata,
 )
 
@@ -1386,8 +1387,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning(err_msg)
             return {}
 
-    def _mark_watched(self, video_id, video_info):
-        playback_url = video_info.get('videostats_playback_base_url', [None])[0]
+    def _mark_watched(self, video_id, video_info, player_response):
+        playback_url = url_or_none(try_get(
+            player_response,
+            lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']) or try_get(
+            video_info, lambda x: x['videostats_playback_base_url'][0]))
         if not playback_url:
             return
         parsed_playback_url = compat_urlparse.urlparse(playback_url)
@@ -2122,7 +2126,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         self._sort_formats(formats)
 
-        self.mark_watched(video_id, video_info)
+        self.mark_watched(video_id, video_info, player_response)
 
         return {
             'id': video_id,

From c8b37510868b20b0829583557899d70db2ea6243 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 16 Dec 2018 14:28:40 +0100
Subject: [PATCH 244/510] [vrv] fix initial state extraction

---
 youtube_dl/extractor/vrv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index ac0819c7c..483a3be3a 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -120,8 +120,10 @@ class VRVIE(VRVBaseIE):
             url, video_id,
             headers=self.geo_verification_headers())
         media_resource = self._parse_json(self._search_regex(
-            r'window\.__INITIAL_STATE__\s*=\s*({.+?})</script>',
-            webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
+            [
+                r'window\.__INITIAL_STATE__\s*=\s*({.+?})(?:</script>|;)',
+                r'window\.__INITIAL_STATE__\s*=\s*({.+})'
+            ], webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
 
         video_data = media_resource.get('json')
         if not video_data:

From 90046d77616f8f24e3716c64443118a9dcbec996 Mon Sep 17 00:00:00 2001
From: yonaikerlol <lawlietrs7@gmail.com>
Date: Sun, 16 Dec 2018 17:10:36 -0400
Subject: [PATCH 245/510] [iprima] Relax _VALID_URL (closes #18515)

---
 youtube_dl/extractor/iprima.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 11a6629d2..11bbeb592 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:play|prima|www)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_BYPASS = False
 
     _TESTS = [{
@@ -44,6 +44,21 @@ class IPrimaIE(InfoExtractor):
     }, {
         'url': 'http://www.iprima.cz/filmy/desne-rande',
         'only_matching': True,
+    }, {
+        'url': 'https://zoom.iprima.cz/10-nejvetsich-tajemstvi-zahad/posvatna-mista-a-stavby',
+        'only_matching': True,
+    }, {
+        'url': 'https://krimi.iprima.cz/mraz-0/sebevrazdy',
+        'only_matching': True,
+    }, {
+        'url': 'https://cool.iprima.cz/derava-silnice-nevadi',
+        'only_matching': True,
+    }, {
+        'url': 'https://love.iprima.cz/laska-az-za-hrob/slib-dany-bratrovi',
+        'only_matching': True,
+    }, {
+        'url': 'https://autosalon.iprima.cz/motorsport/7-epizoda-1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 252e172dea96b90191682afac837535de4d33107 Mon Sep 17 00:00:00 2001
From: Tim Landscheidt <tim@tim-landscheidt.de>
Date: Sun, 16 Dec 2018 21:29:12 +0000
Subject: [PATCH 246/510] [acast] Add support for embed.acast.com

---
 youtube_dl/extractor/acast.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 6d846ea7a..1fbff705d 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -17,25 +17,8 @@ from ..utils import (
 
 class ACastIE(InfoExtractor):
     IE_NAME = 'acast'
-    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:(?:embed|www)\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
     _TESTS = [{
-        # test with one bling
-        'url': 'https://www.acast.com/condenasttraveler/-where-are-you-taipei-101-taiwan',
-        'md5': 'ada3de5a1e3a2a381327d749854788bb',
-        'info_dict': {
-            'id': '57de3baa-4bb0-487e-9418-2692c1277a34',
-            'ext': 'mp3',
-            'title': '"Where Are You?": Taipei 101, Taiwan',
-            'description': 'md5:a0b4ef3634e63866b542e5b1199a1a0e',
-            'timestamp': 1196172000,
-            'upload_date': '20071127',
-            'duration': 211,
-            'creator': 'Concierge',
-            'series': 'Condé Nast Traveler Podcast',
-            'episode': '"Where Are You?": Taipei 101, Taiwan',
-        }
-    }, {
-        # test with multiple blings
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
         'md5': 'a02393c74f3bdb1801c3ec2695577ce0',
         'info_dict': {
@@ -50,6 +33,9 @@ class ACastIE(InfoExtractor):
             'series': 'Spår',
             'episode': '2. Raggarmordet - Röster ur det förflutna',
         }
+    }, {
+        'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 50a498a68e2f6754f5b26e5ad1f6eabfd9aeeb14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Dec 2018 04:32:59 +0700
Subject: [PATCH 247/510] [acast] Extend _VALID_URL

---
 youtube_dl/extractor/acast.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 1fbff705d..b32f74a37 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -17,7 +17,14 @@ from ..utils import (
 
 class ACastIE(InfoExtractor):
     IE_NAME = 'acast'
-    _VALID_URL = r'https?://(?:(?:embed|www)\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:(?:embed|www)\.)?acast\.com/|
+                            play\.acast\.com/s/
+                        )
+                        (?P<channel>[^/]+)/(?P<id>[^/#?]+)
+                    '''
     _TESTS = [{
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
         'md5': 'a02393c74f3bdb1801c3ec2695577ce0',
@@ -36,6 +43,9 @@ class ACastIE(InfoExtractor):
     }, {
         'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
         'only_matching': True,
+    }, {
+        'url': 'https://play.acast.com/s/rattegangspodden/s04e09-styckmordet-i-helenelund-del-22',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 1c82122741783adf6653df25fa81ef0f95a22279 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Dec 2018 04:51:57 +0700
Subject: [PATCH 248/510] [ard:beta] Relax _VALID_URL (closes #18441)

---
 youtube_dl/extractor/ard.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 2e1536a1b..a5df7f0f9 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -56,6 +56,10 @@ class ARDMediathekIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
+
     def _extract_media_info(self, media_info_url, webpage, video_id):
         media_info = self._download_json(
             media_info_url, video_id, 'Downloading media JSON')
@@ -296,7 +300,7 @@ class ARDIE(InfoExtractor):
 
 
 class ARDBetaMediathekIE(InfoExtractor):
-    _VALID_URL = r'https://beta\.ardmediathek\.de/[a-z]+/player/(?P<video_id>[a-zA-Z0-9]+)/(?P<display_id>[^/?#]+)'
+    _VALID_URL = r'https://(?:beta|www)\.ardmediathek\.de/[^/]+/(?:player|live)/(?P<video_id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^/?#]+))?'
     _TESTS = [{
         'url': 'https://beta.ardmediathek.de/ard/player/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE/die-robuste-roswita',
         'md5': '2d02d996156ea3c397cfc5036b5d7f8f',
@@ -310,12 +314,18 @@ class ARDBetaMediathekIE(InfoExtractor):
             'upload_date': '20180826',
             'ext': 'mp4',
         },
+    }, {
+        'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('video_id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or video_id
 
         webpage = self._download_webpage(url, display_id)
         data_json = self._search_regex(r'window\.__APOLLO_STATE__\s*=\s*(\{.*);\n', webpage, 'json')

From 752582183a1942b12880139039137b3a60962611 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Dec 2018 05:29:59 +0700
Subject: [PATCH 249/510] [ard:beta] Improve extraction robustness, fix
 subtitles extraction, improve geo restricted videos extraction

---
 youtube_dl/extractor/ard.py | 58 +++++++++++++++++++++++++------------
 1 file changed, 40 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index a5df7f0f9..8adae4644 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -8,13 +8,16 @@ from .generic import GenericIE
 from ..utils import (
     determine_ext,
     ExtractorError,
-    qualities,
     int_or_none,
     parse_duration,
+    qualities,
+    str_or_none,
+    try_get,
     unified_strdate,
-    xpath_text,
+    unified_timestamp,
     update_url_query,
     url_or_none,
+    xpath_text,
 )
 from ..compat import compat_etree_fromstring
 
@@ -336,43 +339,62 @@ class ARDBetaMediathekIE(InfoExtractor):
             'display_id': display_id,
         }
         formats = []
+        subtitles = {}
+        geoblocked = False
         for widget in data.values():
-            if widget.get('_geoblocked'):
-                raise ExtractorError('This video is not available due to geoblocking', expected=True)
-
+            if widget.get('_geoblocked') is True:
+                geoblocked = True
             if '_duration' in widget:
-                res['duration'] = widget['_duration']
+                res['duration'] = int_or_none(widget['_duration'])
             if 'clipTitle' in widget:
                 res['title'] = widget['clipTitle']
             if '_previewImage' in widget:
                 res['thumbnail'] = widget['_previewImage']
             if 'broadcastedOn' in widget:
-                res['upload_date'] = unified_strdate(widget['broadcastedOn'])
+                res['timestamp'] = unified_timestamp(widget['broadcastedOn'])
             if 'synopsis' in widget:
                 res['description'] = widget['synopsis']
-            if '_subtitleUrl' in widget:
-                res['subtitles'] = {'de': [{
+            subtitle_url = url_or_none(widget.get('_subtitleUrl'))
+            if subtitle_url:
+                subtitles.setdefault('de', []).append({
                     'ext': 'ttml',
-                    'url': widget['_subtitleUrl'],
-                }]}
+                    'url': subtitle_url,
+                })
             if '_quality' in widget:
-                format_url = widget['_stream']['json'][0]
-
-                if format_url.endswith('.f4m'):
+                format_url = url_or_none(try_get(
+                    widget, lambda x: x['_stream']['json'][0]))
+                if not format_url:
+                    continue
+                ext = determine_ext(format_url)
+                if ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         format_url + '?hdcore=3.11.0',
                         video_id, f4m_id='hds', fatal=False))
-                elif format_url.endswith('m3u8'):
+                elif ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                        format_url, video_id, 'mp4', m3u8_id='hls',
+                        fatal=False))
                 else:
+                    # HTTP formats are not available when geoblocked is True,
+                    # other formats are fine though
+                    if geoblocked:
+                        continue
+                    quality = str_or_none(widget.get('_quality'))
                     formats.append({
-                        'format_id': 'http-' + widget['_quality'],
+                        'format_id': ('http-' + quality) if quality else 'http',
                         'url': format_url,
                         'preference': 10,  # Plain HTTP, that's nice
                     })
 
+        if not formats and geoblocked:
+            self.raise_geo_restricted(
+                msg='This video is not available due to geoblocking',
+                countries=['DE'])
+
         self._sort_formats(formats)
-        res['formats'] = formats
+        res.update({
+            'subtitles': subtitles,
+            'formats': formats,
+        })
 
         return res

From cbb3e4b14f59fe2dba5983f7e55308eba2ea025b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Dec 2018 05:34:55 +0700
Subject: [PATCH 250/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9aef67b52..291ae42bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Extractors
+* [ard:beta] Improve geo restricted videos extraction
+* [ard:beta] Fix subtitles extraction
+* [ard:beta] Improve extraction robustness
+* [ard:beta] Relax URL regular expression (#18441)
+* [acast] Add support for embed.acast.com and play.acast.com (#18483)
+* [iprima] Relax URL regular expression (#18515, #18540)
+* [vrv] Fix initial state extraction (#18553)
+* [youtube] Fix mark watched (#18546)
++ [safari] Add support for learning.oreilly.com (#18510)
+* [youtube] Fix multifeed extraction (#18531)
+* [lecturio] Improve subtitles extraction (#18488)
+* [uol] Fix format URL extraction (#18480)
++ [ard:mediathek] Add support for classic.ardmediathek.de (#18473)
+
+
 version 2018.12.09
 
 Core

From 4cee62ade0d991eedb2feae927c44370be3c389e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Dec 2018 05:37:50 +0700
Subject: [PATCH 251/510] release 2018.12.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 CONTRIBUTING.md           | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1ccf410a7..b84559932 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.09*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.17*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.12.09
+[debug] youtube-dl version 2018.12.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index bbcb78808..cba87190d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -152,7 +152,7 @@ After you have ensured this site is distributing its content legally, you can fo
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
 9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
@@ -173,7 +173,7 @@ Extractors are very fragile by nature since they depend on the layout of the sou
 
 ### Mandatory and optional metafields
 
-For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L75-L257) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
 
  - `id` (media identifier)
  - `title` (media title)
@@ -181,7 +181,7 @@ For extraction to work youtube-dl relies on metadata your extractor extracts and
 
 In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` as mandatory. Thus the aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken.
 
-[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+[Any field](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
 #### Example
 
diff --git a/ChangeLog b/ChangeLog
index 291ae42bb..510013e89 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.12.17
 
 Extractors
 * [ard:beta] Improve geo restricted videos extraction
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d15b9583f..0461aac0c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.12.09'
+__version__ = '2018.12.17'

From 7216e9bff71b4c537bb8d56386d789bb83f921f9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 Dec 2018 16:34:51 +0100
Subject: [PATCH 252/510] [discovery] Add support for Scripps Networks watch
 domains(closes #17947)

---
 youtube_dl/extractor/discovery.py       | 35 +++++++++++++++++--------
 youtube_dl/extractor/scrippsnetworks.py | 29 +++++---------------
 2 files changed, 31 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 3589bd428..44fbc41bb 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -17,16 +17,29 @@ from ..compat import compat_HTTPError
 
 
 class DiscoveryIE(DiscoveryGoBaseIE):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?(?P<site>
-            discovery|
-            investigationdiscovery|
-            discoverylife|
-            animalplanet|
-            ahctv|
-            destinationamerica|
-            sciencechannel|
-            tlc|
-            velocity
+    _VALID_URL = r'''(?x)https?://
+        (?P<site>
+            (?:www\.)?
+                (?:
+                    discovery|
+                    investigationdiscovery|
+                    discoverylife|
+                    animalplanet|
+                    ahctv|
+                    destinationamerica|
+                    sciencechannel|
+                    tlc|
+                    velocity
+                )|
+            watch\.
+                (?:
+                    hgtv|
+                    foodnetwork|
+                    travelchannel|
+                    diynetwork|
+                    cookingchanneltv|
+                    motortrend
+                )
         )\.com(?P<path>/tv-shows/[^/]+/(?:video|full-episode)s/(?P<id>[^./?#]+))'''
     _TESTS = [{
         'url': 'https://www.discovery.com/tv-shows/cash-cab/videos/dave-foley',
@@ -71,7 +84,7 @@ class DiscoveryIE(DiscoveryGoBaseIE):
 
         if not access_token:
             access_token = self._download_json(
-                'https://www.%s.com/anonymous' % site, display_id, query={
+                'https://%s.com/anonymous' % site, display_id, query={
                     'authRel': 'authorization',
                     'client_id': try_get(
                         react_data, lambda x: x['application']['apiClientId'],
diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 4023aeef8..8b3275735 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -19,7 +19,7 @@ class ScrippsNetworksWatchIE(AWSIE):
     _VALID_URL = r'''(?x)
                     https?://
                         watch\.
-                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv|geniuskitchen)\.com/
+                        (?P<site>geniuskitchen)\.com/
                         (?:
                             player\.[A-Z0-9]+\.html\#|
                             show/(?:[^/]+/){2}|
@@ -28,38 +28,23 @@ class ScrippsNetworksWatchIE(AWSIE):
                         (?P<id>\d+)
                     '''
     _TESTS = [{
-        'url': 'http://watch.hgtv.com/show/HGTVE/Best-Ever-Treehouses/2241515/Best-Ever-Treehouses/',
-        'md5': '26545fd676d939954c6808274bdb905a',
+        'url': 'http://watch.geniuskitchen.com/player/3787617/Ample-Hills-Ice-Cream-Bike/',
         'info_dict': {
-            'id': '4173834',
+            'id': '4194875',
             'ext': 'mp4',
-            'title': 'Best Ever Treehouses',
-            'description': "We're searching for the most over the top treehouses.",
+            'title': 'Ample Hills Ice Cream Bike',
+            'description': 'Courtney Rada churns up a signature GK Now ice cream with The Scoopmaster.',
             'uploader': 'ANV',
-            'upload_date': '20170922',
-            'timestamp': 1506056400,
+            'upload_date': '20171011',
+            'timestamp': 1507698000,
         },
         'params': {
             'skip_download': True,
         },
         'add_ie': [AnvatoIE.ie_key()],
-    }, {
-        'url': 'http://watch.diynetwork.com/show/DSAL/Salvage-Dawgs/2656646/Covington-Church/',
-        'only_matching': True,
-    }, {
-        'url': 'http://watch.diynetwork.com/player.HNT.html#2656646',
-        'only_matching': True,
-    }, {
-        'url': 'http://watch.geniuskitchen.com/player/3787617/Ample-Hills-Ice-Cream-Bike/',
-        'only_matching': True,
     }]
 
     _SNI_TABLE = {
-        'hgtv': 'hgtv',
-        'diynetwork': 'diy',
-        'foodnetwork': 'food',
-        'cookingchanneltv': 'cook',
-        'travelchannel': 'trav',
         'geniuskitchen': 'genius',
     }
 

From 386d1fea79e861b11c77a7353c07eb162fbb4dae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Dec 2018 23:43:12 +0700
Subject: [PATCH 253/510] [lecturio] Add support for lecturio.de (closes
 #18562)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/lecturio.py   | 45 +++++++++++++++++++++++++++---
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6a5d12ab1..d72f52e36 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -557,6 +557,7 @@ from .lecture2go import Lecture2GoIE
 from .lecturio import (
     LecturioIE,
     LecturioCourseIE,
+    LecturioDeCourseIE,
 )
 from .leeco import (
     LeIE,
diff --git a/youtube_dl/extractor/lecturio.py b/youtube_dl/extractor/lecturio.py
index 0f1265cdf..24f78d928 100644
--- a/youtube_dl/extractor/lecturio.py
+++ b/youtube_dl/extractor/lecturio.py
@@ -64,8 +64,14 @@ class LecturioBaseIE(InfoExtractor):
 
 
 class LecturioIE(LecturioBaseIE):
-    _VALID_URL = r'https://app\.lecturio\.com/[^/]+/(?P<id>[^/?#&]+)\.lecture'
-    _TEST = {
+    _VALID_URL = r'''(?x)
+                    https://
+                        (?:
+                            app\.lecturio\.com/[^/]+/(?P<id>[^/?#&]+)\.lecture|
+                            (?:www\.)?lecturio\.de/[^/]+/(?P<id_de>[^/?#&]+)\.vortrag
+                        )
+                    '''
+    _TESTS = [{
         'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
         'md5': 'f576a797a5b7a5e4e4bbdfc25a6a6870',
         'info_dict': {
@@ -74,7 +80,10 @@ class LecturioIE(LecturioBaseIE):
             'title': 'Important Concepts and Terms – Introduction to Microbiology',
         },
         'skip': 'Requires lecturio account credentials',
-    }
+    }, {
+        'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
+        'only_matching': True,
+    }]
 
     _CC_LANGS = {
         'German': 'de',
@@ -86,7 +95,8 @@ class LecturioIE(LecturioBaseIE):
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id') or mobj.group('id_de')
 
         webpage = self._download_webpage(
             'https://app.lecturio.com/en/lecture/%s/player.html' % display_id,
@@ -190,3 +200,30 @@ class LecturioCourseIE(LecturioBaseIE):
             'title', default=None)
 
         return self.playlist_result(entries, display_id, title)
+
+
+class LecturioDeCourseIE(LecturioBaseIE):
+    _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
+    _TEST = {
+        'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        entries = []
+        for mobj in re.finditer(
+                r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
+                webpage):
+            lecture_url = urljoin(url, mobj.group('url'))
+            lecture_id = mobj.group('id')
+            entries.append(self.url_result(
+                lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
+
+        title = self._search_regex(
+            r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
+
+        return self.playlist_result(entries, display_id, title)

From cfd13c4c45f63c408a3e53ac31561c286c9a3acc Mon Sep 17 00:00:00 2001
From: Daan van Vugt <daanvanvugt@gmail.com>
Date: Mon, 17 Dec 2018 18:03:00 +0100
Subject: [PATCH 254/510] [mediasite] Relax _VALID_URL

---
 youtube_dl/extractor/mediasite.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dl/extractor/mediasite.py
index 84876b883..de37df0c6 100644
--- a/youtube_dl/extractor/mediasite.py
+++ b/youtube_dl/extractor/mediasite.py
@@ -21,7 +21,7 @@ from ..utils import (
 
 
 class MediasiteIE(InfoExtractor):
-    _VALID_URL = r'(?xi)https?://[^/]+/Mediasite/Play/(?P<id>[0-9a-f]{32,34})(?P<query>\?[^#]+|)'
+    _VALID_URL = r'(?xi)https?://[^/]+/Mediasite/(?:Play|Showcase/livebroadcast/Presentation)/(?P<id>[0-9a-f]{32,34})(?P<query>\?[^#]+|)'
     _TESTS = [
         {
             'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
@@ -84,7 +84,11 @@ class MediasiteIE(InfoExtractor):
                 'timestamp': 1333983600,
                 'duration': 7794,
             }
-        }
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Showcase/livebroadcast/Presentation/ada7020854f743c49fbb45c9ec7dbb351d',
+            'only_matching': True,
+        },
     ]
 
     # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])

From 4ee184545455d1ebf4a9f97f86ade2c4a3b3e03b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Dec 2018 01:55:13 +0700
Subject: [PATCH 255/510] [mediasite] Extend _VALID_URL even more

---
 youtube_dl/extractor/mediasite.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dl/extractor/mediasite.py
index de37df0c6..ef9628e65 100644
--- a/youtube_dl/extractor/mediasite.py
+++ b/youtube_dl/extractor/mediasite.py
@@ -21,7 +21,7 @@ from ..utils import (
 
 
 class MediasiteIE(InfoExtractor):
-    _VALID_URL = r'(?xi)https?://[^/]+/Mediasite/(?:Play|Showcase/livebroadcast/Presentation)/(?P<id>[0-9a-f]{32,34})(?P<query>\?[^#]+|)'
+    _VALID_URL = r'(?xi)https?://[^/]+/Mediasite/(?:Play|Showcase/(?:default|livebroadcast)/Presentation)/(?P<id>[0-9a-f]{32,34})(?P<query>\?[^#]+|)'
     _TESTS = [
         {
             'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
@@ -89,6 +89,10 @@ class MediasiteIE(InfoExtractor):
             'url': 'https://collegerama.tudelft.nl/Mediasite/Showcase/livebroadcast/Presentation/ada7020854f743c49fbb45c9ec7dbb351d',
             'only_matching': True,
         },
+        {
+            'url': 'https://mediasite.ntnu.no/Mediasite/Showcase/default/Presentation/7d8b913259334b688986e970fae6fcb31d',
+            'only_matching': True,
+        },
     ]
 
     # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])

From 65e29cdac3f3ee14e07c200293438426b1135719 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 18 Dec 2018 22:46:19 +0100
Subject: [PATCH 256/510] [twitter] pass referer with card request(closes
 #18579)

---
 youtube_dl/extractor/twitter.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index de41065d6..41d0b6be8 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -171,7 +171,8 @@ class TwitterCardIE(TwitterBaseIE):
             urls.append('https://twitter.com/i/videos/' + video_id)
 
         for u in urls:
-            webpage = self._download_webpage(u, video_id)
+            webpage = self._download_webpage(
+                u, video_id, headers={'Referer': 'https://twitter.com/'})
 
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',

From 904bb599be0fcf5fa861f17eee757c3c9494208b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 19 Dec 2018 22:22:10 +0200
Subject: [PATCH 257/510] [README.md] Add flake8 instructions

---
 README.md | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index a1d2904c0..b3c39bf66 100644
--- a/README.md
+++ b/README.md
@@ -1025,15 +1025,19 @@ After you have ensured this site is distributing its content legally, you can fo
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
-9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
+8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](http://flake8.pycqa.org/en/latest/index.html#quickstart):
+
+        $ flake8 youtube_dl/extractor/yourextractor.py
+
+9. Make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
+10. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 

From 835e45abab88e2a1661b86be255e06323e88ec7a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 19 Dec 2018 22:07:37 +0100
Subject: [PATCH 258/510] [crackle] extract ism and http formats

---
 youtube_dl/extractor/crackle.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 8dd9d6687..f73ef6b63 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -48,6 +48,21 @@ class CrackleIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _MEDIA_FILE_SLOTS = {
+        '360p.mp4': {
+            'width': 640,
+            'height': 360,
+        },
+        '480p.mp4': {
+            'width': 768,
+            'height': 432,
+        },
+        '480p_1mbps.mp4': {
+            'width': 852,
+            'height': 480,
+        },
+    }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -95,6 +110,20 @@ class CrackleIE(InfoExtractor):
                 elif ext == 'mpd':
                     formats.extend(self._extract_mpd_formats(
                         format_url, video_id, mpd_id='dash', fatal=False))
+                elif format_url.endswith('.ism/Manifest'):
+                    formats.extend(self._extract_ism_formats(
+                        format_url, video_id, ism_id='mss', fatal=False))
+                else:
+                    mfs_path = e.get('Type')
+                    mfs_info = self._MEDIA_FILE_SLOTS.get(mfs_path)
+                    if not mfs_info:
+                        continue
+                    formats.append({
+                        'url': format_url,
+                        'format_id': 'http-' + mfs_path.split('.')[0],
+                        'width': mfs_info['width'],
+                        'height': mfs_info['height'],
+                    })
             self._sort_formats(formats)
 
             description = media.get('Description')

From e1a0628797e2dc7e5e032f853ab4ddc6a7a08020 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 20 Dec 2018 23:22:51 +0100
Subject: [PATCH 259/510] [liveleak] add support for another embed type and
 restore original format extraction

---
 youtube_dl/extractor/liveleak.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 26671753c..22a067e40 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -87,7 +87,7 @@ class LiveLeakIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
-            r'<iframe[^>]+src="(https?://(?:\w+\.)?liveleak\.com/ll_embed\?[^"]*[if]=[\w_]+[^"]+)"',
+            r'<iframe[^>]+src="(https?://(?:\w+\.)?liveleak\.com/ll_embed\?[^"]*[ift]=[\w_]+[^"]+)"',
             webpage)
 
     def _real_extract(self, url):
@@ -120,13 +120,27 @@ class LiveLeakIE(InfoExtractor):
             }
 
         for idx, info_dict in enumerate(entries):
+            formats = []
             for a_format in info_dict['formats']:
                 if not a_format.get('height'):
                     a_format['height'] = int_or_none(self._search_regex(
                         r'([0-9]+)p\.mp4', a_format['url'], 'height label',
                         default=None))
+                formats.append(a_format)
 
-            self._sort_formats(info_dict['formats'])
+                # Removing '.*.mp4' gives the raw video, which is essentially
+                # the same video without the LiveLeak logo at the top (see
+                # https://github.com/rg3/youtube-dl/pull/4768)
+                orig_url = re.sub(r'\.mp4\.[^.]+', '', a_format['url'])
+                if a_format['url'] != orig_url:
+                    format_id = a_format.get('format_id')
+                    formats.append({
+                        'format_id': 'original' + ('-' + format_id if format_id else ''),
+                        'url': orig_url,
+                        'preference': 1,
+                    })
+            self._sort_formats(formats)
+            info_dict['formats'] = formats
 
             # Don't append entry ID for one-video pages to keep backward compatibility
             if len(entries) > 1:
@@ -146,7 +160,7 @@ class LiveLeakIE(InfoExtractor):
 
 
 class LiveLeakEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?liveleak\.com/ll_embed\?.*?\b(?P<kind>[if])=(?P<id>[\w_]+)'
+    _VALID_URL = r'https?://(?:www\.)?liveleak\.com/ll_embed\?.*?\b(?P<kind>[ift])=(?P<id>[\w_]+)'
 
     # See generic.py for actual test cases
     _TESTS = [{
@@ -158,15 +172,14 @@ class LiveLeakEmbedIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        kind, video_id = mobj.group('kind', 'id')
+        kind, video_id = re.match(self._VALID_URL, url).groups()
 
         if kind == 'f':
             webpage = self._download_webpage(url, video_id)
             liveleak_url = self._search_regex(
-                r'logourl\s*:\s*(?P<q1>[\'"])(?P<url>%s)(?P=q1)' % LiveLeakIE._VALID_URL,
+                r'(?:logourl\s*:\s*|window\.open\()(?P<q1>[\'"])(?P<url>%s)(?P=q1)' % LiveLeakIE._VALID_URL,
                 webpage, 'LiveLeak URL', group='url')
-        elif kind == 'i':
-            liveleak_url = 'http://www.liveleak.com/view?i=%s' % video_id
+        else:
+            liveleak_url = 'http://www.liveleak.com/view?%s=%s' % (kind, video_id)
 
         return self.url_result(liveleak_url, ie=LiveLeakIE.ie_key())

From 4273caf5c7e4a79783106938802c1f4e1aa9c950 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Dec 2018 16:38:16 +0700
Subject: [PATCH 260/510] [youtube] Extend html5 player regex (closes #17516)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 44c25c11c..906774875 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1834,7 +1834,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             else:
                                 player_version = self._search_regex(
                                     [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
-                                     r'(?:www|player)-([^/]+)(?:/[a-z]{2}_[A-Z]{2})?/base\.js'],
+                                     r'(?:www|player(?:_ias)?)-([^/]+)(?:/[a-z]{2}_[A-Z]{2})?/base\.js'],
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From 63529e935cf5f87e6080607ef9d9196fe435e092 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Dec 2018 16:57:10 +0700
Subject: [PATCH 261/510] [youtube] Relax html5 player regexes (closes #18465,
 closes #18466)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 906774875..954853e00 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1105,7 +1105,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)
@@ -1834,7 +1834,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             else:
                                 player_version = self._search_regex(
                                     [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
-                                     r'(?:www|player(?:_ias)?)-([^/]+)(?:/[a-z]{2}_[A-Z]{2})?/base\.js'],
+                                     r'(?:www|player(?:_ias)?)-([^/]+)(?:/[a-z]{2,3}_[A-Z]{2})?/base\.js'],
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From 825cd268a3a02e80a9c36dada664d25074243d26 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 26 Dec 2018 09:30:48 +0100
Subject: [PATCH 262/510] [youtube] detect DRM protected videos(#1774)

---
 youtube_dl/extractor/youtube.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 954853e00..4b06cb597 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1077,6 +1077,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
             'only_matching': True,
         },
+        {
+            # DRM protected
+            'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
+            'only_matching': True,
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1673,6 +1678,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     '"token" parameter not in video info for unknown reason',
                     video_id=video_id)
 
+        if video_info.get('license_info'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
         video_details = try_get(
             player_response, lambda x: x['videoDetails'], dict) or {}
 

From 140a13f5de1ff8e6bf7ba0d27b38063a2b20fa33 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 26 Dec 2018 10:55:28 +0100
Subject: [PATCH 263/510] [youtube] extract more format metadata

---
 youtube_dl/extractor/youtube.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4b06cb597..ac3b4fc17 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1794,6 +1794,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 'height': int_or_none(width_height[1]),
                             }
             q = qualities(['small', 'medium', 'hd720'])
+            streaming_formats = try_get(player_response, lambda x: x['streamingData']['formats'], list)
+            if streaming_formats:
+                for fmt in streaming_formats:
+                    itag = str_or_none(fmt.get('itag'))
+                    if not itag:
+                        continue
+                    quality = fmt.get('quality')
+                    quality_label = fmt.get('qualityLabel') or quality
+                    formats_spec[itag] = {
+                        'asr': int_or_none(fmt.get('audioSampleRate')),
+                        'filesize': int_or_none(fmt.get('contentLength')),
+                        'format_note': quality_label,
+                        'fps': int_or_none(fmt.get('fps')),
+                        'height': int_or_none(fmt.get('height')),
+                        'quality': q(quality),
+                        # bitrate for itag 43 is always 2147483647
+                        'tbr': float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) if itag != '43' else None,
+                        'width': int_or_none(fmt.get('width')),
+                    }
             formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)
@@ -1876,7 +1895,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 filesize = int_or_none(url_data.get(
                     'clen', [None])[0]) or _extract_filesize(url)
 
-                quality = url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0]
+                quality = url_data.get('quality', [None])[0]
 
                 more_fields = {
                     'filesize': filesize,
@@ -1884,7 +1903,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'width': width,
                     'height': height,
                     'fps': int_or_none(url_data.get('fps', [None])[0]),
-                    'format_note': quality,
+                    'format_note': url_data.get('quality_label', [None])[0] or quality,
                     'quality': q(quality),
                 }
                 for key, value in more_fields.items():

From c2dd2dc086653f46defe0bc6bee9b5e339e67dea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Dec 2018 19:57:01 +0700
Subject: [PATCH 264/510] [youtube] Unescape HTML for series (closes #18641)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ac3b4fc17..539ff6ff2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2043,7 +2043,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
             video_webpage)
         if m_episode:
-            series = m_episode.group('series')
+            series = unescapeHTML(m_episode.group('series'))
             season_number = int(m_episode.group('season'))
             episode_number = int(m_episode.group('episode'))
         else:

From e4d51e751e361faaedaa50baf37501106459ab73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Dec 2018 20:59:53 +0700
Subject: [PATCH 265/510] [beeg] Fix extraction (closes #18610, closes #18626)

---
 youtube_dl/extractor/beeg.py | 75 ++++++------------------------------
 1 file changed, 11 insertions(+), 64 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index bf22a41b7..1086d7632 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,15 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_chr,
-    compat_ord,
-    compat_urllib_parse_unquote,
-)
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    parse_iso8601,
-    urljoin,
+    unified_timestamp,
 )
 
 
@@ -36,29 +31,9 @@ class BeegIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        cpl_url = self._search_regex(
-            r'<script[^>]+src=(["\'])(?P<url>(?:/static|(?:https?:)?//static\.beeg\.com)/cpl/\d+\.js.*?)\1',
-            webpage, 'cpl', default=None, group='url')
-
-        cpl_url = urljoin(url, cpl_url)
-
-        beeg_version, beeg_salt = [None] * 2
-
-        if cpl_url:
-            cpl = self._download_webpage(
-                self._proto_relative_url(cpl_url), video_id,
-                'Downloading cpl JS', fatal=False)
-            if cpl:
-                beeg_version = int_or_none(self._search_regex(
-                    r'beeg_version\s*=\s*([^\b]+)', cpl,
-                    'beeg version', default=None)) or self._search_regex(
-                    r'/(\d+)\.js', cpl_url, 'beeg version', default=None)
-                beeg_salt = self._search_regex(
-                    r'beeg_salt\s*=\s*(["\'])(?P<beeg_salt>.+?)\1', cpl, 'beeg salt',
-                    default=None, group='beeg_salt')
-
-        beeg_version = beeg_version or '2185'
-        beeg_salt = beeg_salt or 'pmweAkq8lAYKdfWcFCUj0yoVgoPlinamH5UE1CB3H'
+        beeg_version = self._search_regex(
+            r'beeg_version\s*=\s*([\da-zA-Z_-]+)', webpage, 'beeg version',
+            default='1546225636701')
 
         for api_path in ('', 'api.'):
             video = self._download_json(
@@ -68,37 +43,6 @@ class BeegIE(InfoExtractor):
             if video:
                 break
 
-        def split(o, e):
-            def cut(s, x):
-                n.append(s[:x])
-                return s[x:]
-            n = []
-            r = len(o) % e
-            if r > 0:
-                o = cut(o, r)
-            while len(o) > e:
-                o = cut(o, e)
-            n.append(o)
-            return n
-
-        def decrypt_key(key):
-            # Reverse engineered from http://static.beeg.com/cpl/1738.js
-            a = beeg_salt
-            e = compat_urllib_parse_unquote(key)
-            o = ''.join([
-                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)
-                for n in range(len(e))])
-            return ''.join(split(o, 3)[::-1])
-
-        def decrypt_url(encrypted_url):
-            encrypted_url = self._proto_relative_url(
-                encrypted_url.replace('{DATA_MARKERS}', ''), 'https:')
-            key = self._search_regex(
-                r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
-            if not key:
-                return encrypted_url
-            return encrypted_url.replace(key, decrypt_key(key))
-
         formats = []
         for format_id, video_url in video.items():
             if not video_url:
@@ -108,18 +52,20 @@ class BeegIE(InfoExtractor):
             if not height:
                 continue
             formats.append({
-                'url': decrypt_url(video_url),
+                'url': self._proto_relative_url(
+                    video_url.replace('{DATA_MARKERS}', 'data=pc_XX__%s_0' % beeg_version), 'https:'),
                 'format_id': format_id,
                 'height': int(height),
             })
         self._sort_formats(formats)
 
         title = video['title']
-        video_id = video.get('id') or video_id
+        video_id = compat_str(video.get('id') or video_id)
         display_id = video.get('code')
         description = video.get('desc')
+        series = video.get('ps_name')
 
-        timestamp = parse_iso8601(video.get('date'), ' ')
+        timestamp = unified_timestamp(video.get('date'))
         duration = int_or_none(video.get('duration'))
 
         tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None
@@ -129,6 +75,7 @@ class BeegIE(InfoExtractor):
             'display_id': display_id,
             'title': title,
             'description': description,
+            'series': series,
             'timestamp': timestamp,
             'duration': duration,
             'tags': tags,

From 4e1ddc8da9b318873d71d0283838af6aec1bc98d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Dec 2018 21:05:07 +0700
Subject: [PATCH 266/510] [npo:live] Add support for npostart.nl (closes
 #18644)

---
 youtube_dl/extractor/npo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c2cb85a73..5a427c396 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -363,7 +363,7 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(?:/(?P<id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:www\.)?npo(?:start)?\.nl/live(?:/(?P<id>[^/?#&]+))?'
 
     _TESTS = [{
         'url': 'http://www.npo.nl/live/npo-1',
@@ -380,6 +380,9 @@ class NPOLiveIE(NPOBaseIE):
     }, {
         'url': 'http://www.npo.nl/live',
         'only_matching': True,
+    }, {
+        'url': 'https://www.npostart.nl/live/npo-1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 373941c5f0c163bf9f2fbbe7f65ee549a228720f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Dec 2018 23:20:40 +0700
Subject: [PATCH 267/510] [bbc] Add support for another embed pattern (closes
 #18643)

---
 youtube_dl/extractor/bbc.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index abcfa301d..eac9a5a46 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -795,6 +795,15 @@ class BBCIE(BBCCoUkIE):
             'uploader': 'Radio 3',
             'uploader_id': 'bbc_radio_three',
         },
+    }, {
+        'url': 'http://www.bbc.co.uk/learningenglish/chinese/features/lingohack/ep-181227',
+        'info_dict': {
+            'id': 'p06w9tws',
+            'ext': 'mp4',
+            'title': 'md5:2fabf12a726603193a2879a055f72514',
+            'description': 'Learn English words and phrases from this story',
+        },
+        'add_ie': [BBCCoUkIE.ie_key()],
     }]
 
     @classmethod
@@ -945,6 +954,15 @@ class BBCIE(BBCCoUkIE):
         if entries:
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
+        # http://www.bbc.co.uk/learningenglish/chinese/features/lingohack/ep-181227
+        group_id = self._search_regex(
+            r'<div[^>]+\bclass=["\']video["\'][^>]+\bdata-pid=["\'](%s)' % self._ID_REGEX,
+            webpage, 'group id', default=None)
+        if playlist_id:
+            return self.url_result(
+                'https://www.bbc.co.uk/programmes/%s' % group_id,
+                ie=BBCCoUkIE.ie_key())
+
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(
             [r'data-(?:video-player|media)-vpid="(%s)"' % self._ID_REGEX,

From 32ac3d49ae83fe933515096a74068a8033131f4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Dec 2018 23:57:46 +0700
Subject: [PATCH 268/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 510013e89..31458dbd9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Extractors
++ [bbc] Add support for another embed pattern (#18643)
++ [npo:live] Add support for npostart.nl (#18644)
+* [beeg] Fix extraction (#18610, #18626)
+* [youtube] Unescape HTML for series (#18641)
++ [youtube] Extract more format metadata
+* [youtube] Detect DRM protected videos (#1774)
+* [youtube] Relax HTML5 player regular expressions (#18465, #18466)
+* [youtube] Extend HTML5 player regular expression (#17516)
++ [liveleak] Add support for another embed type and restore original
+  format extraction
++ [crackle] Extract ISM and HTTP formats
++ [twitter] Pass Referer with card request (#18579)
+* [mediasite] Extend URL regular expression (#18558)
++ [lecturio] Add support for lecturio.de (#18562)
++ [discovery] Add support for Scripps Networks watch domains (#17947)
+
+
 version 2018.12.17
 
 Extractors

From 9d9daed4647c0a0798adee5a580e9fd95478cf6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Dec 2018 23:59:52 +0700
Subject: [PATCH 269/510] release 2018.12.31

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 CONTRIBUTING.md           | 10 +++++++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    |  1 +
 youtube_dl/version.py     |  2 +-
 5 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b84559932..b2fe6ee6a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.17*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.31*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.31**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.12.17
+[debug] youtube-dl version 2018.12.31
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cba87190d..24617264c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -153,15 +153,19 @@ After you have ensured this site is distributing its content legally, you can fo
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
-9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
+8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](http://flake8.pycqa.org/en/latest/index.html#quickstart):
+
+        $ flake8 youtube_dl/extractor/yourextractor.py
+
+9. Make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
+10. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 
diff --git a/ChangeLog b/ChangeLog
index 31458dbd9..72bfe2110 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.12.31
 
 Extractors
 + [bbc] Add support for another embed pattern (#18643)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 31d20f255..e92064432 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -438,6 +438,7 @@
  - **Lecture2Go**
  - **Lecturio**
  - **LecturioCourse**
+ - **LecturioDeCourse**
  - **LEGO**
  - **Lemonde**
  - **Lenta**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0461aac0c..87de5848d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.12.17'
+__version__ = '2018.12.31'

From 6b688b8942a0bd18b2a3835e69a903f1eeffee2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Jan 2019 18:12:44 +0700
Subject: [PATCH 270/510] [bitchute] Fix extraction (closes #18567)

---
 youtube_dl/extractor/bitchute.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 43b4732aa..aa034355a 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -5,7 +5,10 @@ import itertools
 import re
 
 from .common import InfoExtractor
-from ..utils import urlencode_postdata
+from ..utils import (
+    orderedSet,
+    urlencode_postdata,
+)
 
 
 class BitChuteIE(InfoExtractor):
@@ -43,10 +46,15 @@ class BitChuteIE(InfoExtractor):
             'description', webpage, 'title',
             default=None) or self._og_search_description(webpage)
 
+        format_urls = []
+        for mobj in re.finditer(
+                r'addWebSeed\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage):
+            format_urls.append(mobj.group('url'))
+        format_urls.extend(re.findall(r'as=(https?://[^&"\']+)', webpage))
+
         formats = [
-            {'url': mobj.group('url')}
-            for mobj in re.finditer(
-                r'addWebSeed\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage)]
+            {'url': format_url}
+            for format_url in orderedSet(format_urls)]
         self._sort_formats(formats)
 
         description = self._html_search_regex(

From 38d15ba7f981cc82db91edcbf05ea3844082d045 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Jan 2019 20:31:48 +0700
Subject: [PATCH 271/510] [manyvids] Fix extraction (closes #18604, closes
 #18614)

---
 youtube_dl/extractor/manyvids.py | 62 +++++++++++++++++++++++++++-----
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dl/extractor/manyvids.py
index b94b3c2ab..e8d7163e4 100644
--- a/youtube_dl/extractor/manyvids.py
+++ b/youtube_dl/extractor/manyvids.py
@@ -2,12 +2,18 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    str_to_int,
+    urlencode_postdata,
+)
 
 
 class ManyVidsIE(InfoExtractor):
     _VALID_URL = r'(?i)https?://(?:www\.)?manyvids\.com/video/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
+        # preview video
         'url': 'https://www.manyvids.com/Video/133957/everthing-about-me/',
         'md5': '03f11bb21c52dd12a05be21a5c7dcc97',
         'info_dict': {
@@ -17,7 +23,18 @@ class ManyVidsIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         },
-    }
+    }, {
+        # full video
+        'url': 'https://www.manyvids.com/Video/935718/MY-FACE-REVEAL/',
+        'md5': 'f3e8f7086409e9b470e2643edb96bdcc',
+        'info_dict': {
+            'id': '935718',
+            'ext': 'mp4',
+            'title': 'MY FACE REVEAL',
+            'view_count': int,
+            'like_count': int,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -28,12 +45,41 @@ class ManyVidsIE(InfoExtractor):
             r'data-(?:video-filepath|meta-video)\s*=s*(["\'])(?P<url>(?:(?!\1).)+)\1',
             webpage, 'video URL', group='url')
 
-        title = '%s (Preview)' % self._html_search_regex(
-            r'<h2[^>]+class="m-a-0"[^>]*>([^<]+)', webpage, 'title')
+        title = self._html_search_regex(
+            (r'<span[^>]+class=["\']item-title[^>]+>([^<]+)',
+             r'<h2[^>]+class=["\']h2 m-0["\'][^>]*>([^<]+)'),
+            webpage, 'title', default=None) or self._html_search_meta(
+            'twitter:title', webpage, 'title', fatal=True)
+
+        if any(p in webpage for p in ('preview_videos', '_preview.mp4')):
+            title += ' (Preview)'
+
+        mv_token = self._search_regex(
+            r'data-mvtoken=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+            'mv token', default=None, group='value')
+
+        if mv_token:
+            # Sets some cookies
+            self._download_webpage(
+                'https://www.manyvids.com/includes/ajax_repository/you_had_me_at_hello.php',
+                video_id, fatal=False, data=urlencode_postdata({
+                    'mvtoken': mv_token,
+                    'vid': video_id,
+                }), headers={
+                    'Referer': url,
+                    'X-Requested-With': 'XMLHttpRequest'
+                })
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls')
+        else:
+            formats = [{'url': video_url}]
 
         like_count = int_or_none(self._search_regex(
             r'data-likes=["\'](\d+)', webpage, 'like count', default=None))
-        view_count = int_or_none(self._html_search_regex(
+        view_count = str_to_int(self._html_search_regex(
             r'(?s)<span[^>]+class="views-wrapper"[^>]*>(.+?)</span', webpage,
             'view count', default=None))
 
@@ -42,7 +88,5 @@ class ManyVidsIE(InfoExtractor):
             'title': title,
             'view_count': view_count,
             'like_count': like_count,
-            'formats': [{
-                'url': video_url,
-            }],
+            'formats': formats,
         }

From 3d8eb6beb9aa6545be3184761df12cdcd0cb2944 Mon Sep 17 00:00:00 2001
From: Anarky <ghostanarky+aur@gmail.com>
Date: Sun, 30 Dec 2018 14:30:36 +0100
Subject: [PATCH 272/510] [rmcdecouverte] Update _VALID_URL (closes #18595)

---
 youtube_dl/extractor/rmcdecouverte.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
index e921ca3e6..c44b90274 100644
--- a/youtube_dl/extractor/rmcdecouverte.py
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -10,18 +10,18 @@ from ..compat import (
 
 
 class RMCDecouverteIE(InfoExtractor):
-    _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/mediaplayer-replay.*?\bid=(?P<id>\d+)'
+    _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/.+/program_(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=13502&title=AQUAMEN:LES%20ROIS%20DES%20AQUARIUMS%20:UN%20DELICIEUX%20PROJET',
+        'url': 'https://rmcdecouverte.bfmtv.com/wheeler-dealers-occasions-a-saisir/program_2566/',
         'info_dict': {
-            'id': '5419055995001',
+            'id': '5983675500001',
             'ext': 'mp4',
-            'title': 'UN DELICIEUX PROJET',
-            'description': 'md5:63610df7c8b1fc1698acd4d0d90ba8b5',
+            'title': 'CORVETTE',
+            'description': 'md5:c1e8295521e45ffebf635d6a7658f506',
             'uploader_id': '1969646226001',
-            'upload_date': '20170502',
-            'timestamp': 1493745308,
+            'upload_date': '20181226',
+            'timestamp': 1545861635,
         },
         'params': {
             'skip_download': True,

From d9f1123c08e21c7170b906c7de2562b7293f7a3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Jan 2019 20:49:13 +0700
Subject: [PATCH 273/510] [rmcdecouverte] Improve, bypass geo restriction and
 add support for live (closes #18697)

---
 youtube_dl/extractor/rmcdecouverte.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
index c44b90274..c3623edcc 100644
--- a/youtube_dl/extractor/rmcdecouverte.py
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -1,18 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
 from ..compat import (
     compat_parse_qs,
     compat_urlparse,
 )
+from ..utils import smuggle_url
 
 
 class RMCDecouverteIE(InfoExtractor):
-    _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/.+/program_(?P<id>\d+)'
+    _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/(?:(?:[^/]+/)*program_(?P<id>\d+)|(?P<live_id>mediaplayer-direct))'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://rmcdecouverte.bfmtv.com/wheeler-dealers-occasions-a-saisir/program_2566/',
         'info_dict': {
             'id': '5983675500001',
@@ -27,12 +30,17 @@ class RMCDecouverteIE(InfoExtractor):
             'skip_download': True,
         },
         'skip': 'only available for a week',
-    }
+    }, {
+        # live, geo restricted, bypassable
+        'url': 'https://rmcdecouverte.bfmtv.com/mediaplayer-direct/',
+        'only_matching': True,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1969646226001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id') or mobj.group('live_id')
+        webpage = self._download_webpage(url, display_id)
         brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if brightcove_legacy_url:
             brightcove_id = compat_parse_qs(compat_urlparse.urlparse(
@@ -41,5 +49,7 @@ class RMCDecouverteIE(InfoExtractor):
             brightcove_id = self._search_regex(
                 r'data-video-id=["\'](\d+)', webpage, 'brightcove id')
         return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew',
-            brightcove_id)
+            smuggle_url(
+                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+                {'geo_countries': ['FR']}),
+            'BrightcoveNew', brightcove_id)

From 9b5c8751eeb10fd41c560696d73fec3401055ddc Mon Sep 17 00:00:00 2001
From: iwconfig <snelhingst@gmail.com>
Date: Tue, 1 Jan 2019 15:39:18 +0100
Subject: [PATCH 274/510] [extractors] Add missing age limits

---
 youtube_dl/extractor/cammodels.py  | 2 ++
 youtube_dl/extractor/camtube.py    | 2 ++
 youtube_dl/extractor/camwithher.py | 2 ++
 youtube_dl/extractor/yourporn.py   | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/cammodels.py b/youtube_dl/extractor/cammodels.py
index 79350817f..1eb81b75e 100644
--- a/youtube_dl/extractor/cammodels.py
+++ b/youtube_dl/extractor/cammodels.py
@@ -14,6 +14,7 @@ class CamModelsIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://www.cammodels.com/cam/AutumnKnight/',
         'only_matching': True,
+        'age_limit': 18
     }]
 
     def _real_extract(self, url):
@@ -93,4 +94,5 @@ class CamModelsIE(InfoExtractor):
             'title': self._live_title(user_id),
             'is_live': True,
             'formats': formats,
+            'age_limit': 18
         }
diff --git a/youtube_dl/extractor/camtube.py b/youtube_dl/extractor/camtube.py
index c7d40f849..b3be3bdcf 100644
--- a/youtube_dl/extractor/camtube.py
+++ b/youtube_dl/extractor/camtube.py
@@ -20,6 +20,7 @@ class CamTubeIE(InfoExtractor):
             'duration': 1274,
             'timestamp': 1528018608,
             'upload_date': '20180603',
+            'age_limit': 18
         },
         'params': {
             'skip_download': True,
@@ -66,4 +67,5 @@ class CamTubeIE(InfoExtractor):
             'like_count': like_count,
             'creator': creator,
             'formats': formats,
+            'age_limit': 18
         }
diff --git a/youtube_dl/extractor/camwithher.py b/youtube_dl/extractor/camwithher.py
index afbc5ea26..bbc5205fd 100644
--- a/youtube_dl/extractor/camwithher.py
+++ b/youtube_dl/extractor/camwithher.py
@@ -25,6 +25,7 @@ class CamWithHerIE(InfoExtractor):
             'comment_count': int,
             'uploader': 'MileenaK',
             'upload_date': '20160322',
+            'age_limit': 18,
         },
         'params': {
             'skip_download': True,
@@ -84,4 +85,5 @@ class CamWithHerIE(InfoExtractor):
             'comment_count': comment_count,
             'uploader': uploader,
             'upload_date': upload_date,
+            'age_limit': 18
         }
diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py
index a9951f3b8..432d1fff4 100644
--- a/youtube_dl/extractor/yourporn.py
+++ b/youtube_dl/extractor/yourporn.py
@@ -14,6 +14,7 @@ class YourPornIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'md5:c9f43630bd968267672651ba905a7d35',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'age_limit': 18
         },
     }
 
@@ -38,4 +39,5 @@ class YourPornIE(InfoExtractor):
             'url': video_url,
             'title': title,
             'thumbnail': thumbnail,
+            'age_limit': 18
         }

From 0e713dbb116493181218c38dc03716feda2fa7d9 Mon Sep 17 00:00:00 2001
From: biwubo <45994985+biwubo@users.noreply.github.com>
Date: Tue, 1 Jan 2019 15:48:06 +0000
Subject: [PATCH 275/510] [acast:channel] Add support for play.acast.com
 (closes #18587)

---
 youtube_dl/extractor/acast.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index b32f74a37..c4362be88 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -79,17 +79,27 @@ class ACastIE(InfoExtractor):
 
 class ACastChannelIE(InfoExtractor):
     IE_NAME = 'acast:channel'
-    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
-    _TEST = {
-        'url': 'https://www.acast.com/condenasttraveler',
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?acast\.com/|
+                            play\.acast\.com/s/
+                        )
+                        (?P<id>[^/#?]+)
+                    '''
+    _TESTS = [{
+        'url': 'https://www.acast.com/todayinfocus',
         'info_dict': {
-            'id': '50544219-29bb-499e-a083-6087f4cb7797',
-            'title': 'Condé Nast Traveler Podcast',
-            'description': 'md5:98646dee22a5b386626ae31866638fbd',
+            'id': '4efc5294-5385-4847-98bd-519799ce5786',
+            'title': 'Today in Focus',
+            'description': 'md5:9ba5564de5ce897faeb12963f4537a64',
         },
-        'playlist_mincount': 20,
-    }
-    _API_BASE_URL = 'https://www.acast.com/api/'
+        'playlist_mincount': 35,
+    }, {
+        'url': 'http://play.acast.com/s/ft-banking-weekly',
+        'only_matching': True,
+    }]
+    _API_BASE_URL = 'https://play.acast.com/api/'
     _PAGE_SIZE = 10
 
     @classmethod
@@ -102,7 +112,7 @@ class ACastChannelIE(InfoExtractor):
             channel_slug, note='Download page %d of channel data' % page)
         for cast in casts:
             yield self.url_result(
-                'https://www.acast.com/%s/%s' % (channel_slug, cast['url']),
+                'https://play.acast.com/s/%s/%s' % (channel_slug, cast['url']),
                 'ACast', cast['id'])
 
     def _real_extract(self, url):

From 696f4e41149a8f91e963d33e2f6ea9bc835f2260 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 1 Jan 2019 23:13:39 +0700
Subject: [PATCH 276/510] [README.md] Add more guide lines for regular
 expressions

---
 README.md | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b3c39bf66..bdc5faeec 100644
--- a/README.md
+++ b/README.md
@@ -1133,11 +1133,33 @@ title = meta.get('title') or self._og_search_title(webpage)
 
 This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
 
-### Make regular expressions flexible
+### Regular expressions
 
-When using regular expressions try to write them fuzzy and flexible.
+#### Don't capture groups you don't use
+
+Capturing group must be an indication that it's used somewhere in the code. Any group that is not used must be non capturing.
+
+##### Example
+
+Don't capture id attribute name here since you can't use it for anything anyway.
+
+Correct:
+
+```python
+r'(?:id|ID)=(?P<id>\d+)'
+```
+
+Incorrect:
+```python
+r'(id|ID)=(?P<id>\d+)'
+```
+
+
+#### Make regular expressions relaxed and flexible
+
+When using regular expressions try to write them fuzzy, relaxed and flexible, skipping insignificant parts that are more likely to change, allowing both single and double quotes for quoted values and so on.
  
-#### Example
+##### Example
 
 Say you need to extract `title` from the following HTML code:
 

From 1d803085d730bf2aaf54e5cea2362133b3fdb831 Mon Sep 17 00:00:00 2001
From: v-delta <45652398+v-delta@users.noreply.github.com>
Date: Tue, 1 Jan 2019 17:26:59 +0100
Subject: [PATCH 277/510] [yourporn] Fix extraction

---
 youtube_dl/extractor/yourporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py
index 432d1fff4..c8dc29bd8 100644
--- a/youtube_dl/extractor/yourporn.py
+++ b/youtube_dl/extractor/yourporn.py
@@ -27,7 +27,7 @@ class YourPornIE(InfoExtractor):
             self._search_regex(
                 r'data-vnfo=(["\'])(?P<data>{.+?})\1', webpage, 'data info',
                 group='data'),
-            video_id)[video_id]).replace('/cdn/', '/cdn2/')
+            video_id)[video_id]).replace('/cdn/', '/cdn3/')
 
         title = (self._search_regex(
             r'<[^>]+\bclass=["\']PostEditTA[^>]+>([^<]+)', webpage, 'title',

From 8437f5089f2fac85a249c4368c2ea1415955d8b0 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 1 Jan 2019 23:50:02 +0700
Subject: [PATCH 278/510] [README.md] Add long lines policy to coding
 conventions

---
 README.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/README.md b/README.md
index bdc5faeec..901595444 100644
--- a/README.md
+++ b/README.md
@@ -1192,6 +1192,25 @@ title = self._search_regex(
     webpage, 'title', group='title')
 ```
 
+### Long lines policy
+
+There is a soft limit to keep lines of code under 80 characters long. This means it should be respected if possible and if it does not make readability and code maintenance worse.
+
+For example, you should **never** split long string literals like URLs or some other often copied entities over multiple lines to fit this limit:
+
+Correct:
+
+```python
+'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
+```
+
+Incorrect:
+
+```python
+'https://www.youtube.com/watch?v=FqZTN594JQw&list='
+'PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
+```
+
 ### Use safe conversion functions
 
 Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.

From d226c560a6250d03ff5f27d5b078f894d448a0b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Tue, 1 Jan 2019 18:56:05 +0200
Subject: [PATCH 279/510] Refactor code to use url_result

---
 youtube_dl/extractor/audiomack.py  |  2 +-
 youtube_dl/extractor/cnn.py        | 12 ++----------
 youtube_dl/extractor/freespeech.py |  7 ++-----
 youtube_dl/extractor/generic.py    |  5 +----
 youtube_dl/extractor/livestream.py |  5 +----
 youtube_dl/extractor/savefrom.py   |  7 ++-----
 youtube_dl/extractor/ted.py        |  6 ++----
 youtube_dl/extractor/testurl.py    |  6 +-----
 youtube_dl/extractor/wimp.py       |  6 +-----
 9 files changed, 13 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 62049b921..cc7771354 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -62,7 +62,7 @@ class AudiomackIE(InfoExtractor):
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # if so, pass the work off to the soundcloud extractor
         if SoundcloudIE.suitable(api_response['url']):
-            return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}
+            return self.url_result(api_response['url'], SoundcloudIE.ie_key())
 
         return {
             'id': compat_str(api_response.get('id', album_url_tag)),
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 5fc311f53..774b71055 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -119,11 +119,7 @@ class CNNBlogsIE(InfoExtractor):
     def _real_extract(self, url):
         webpage = self._download_webpage(url, url_basename(url))
         cnn_url = self._html_search_regex(r'data-url="(.+?)"', webpage, 'cnn url')
-        return {
-            '_type': 'url',
-            'url': cnn_url,
-            'ie_key': CNNIE.ie_key(),
-        }
+        return self.url_result(cnn_url, CNNIE.ie_key())
 
 
 class CNNArticleIE(InfoExtractor):
@@ -145,8 +141,4 @@ class CNNArticleIE(InfoExtractor):
     def _real_extract(self, url):
         webpage = self._download_webpage(url, url_basename(url))
         cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
-        return {
-            '_type': 'url',
-            'url': 'http://cnn.com/video/?/video/' + cnn_url,
-            'ie_key': CNNIE.ie_key(),
-        }
+        return self.url_result('http://cnn.com/video/?/video/' + cnn_url, CNNIE.ie_key())
diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index 486a49c05..ea9c3e317 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 
 
 class FreespeechIE(InfoExtractor):
@@ -27,8 +28,4 @@ class FreespeechIE(InfoExtractor):
             r'data-video-url="([^"]+)"',
             webpage, 'youtube url')
 
-        return {
-            '_type': 'url',
-            'url': youtube_url,
-            'ie_key': 'Youtube',
-        }
+        return self.url_result(youtube_url, YoutubeIE.ie_key())
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 65b482333..067de28cd 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2197,10 +2197,7 @@ class GenericIE(InfoExtractor):
 
     def _real_extract(self, url):
         if url.startswith('//'):
-            return {
-                '_type': 'url',
-                'url': self.http_scheme() + url,
-            }
+            return self.url_result(self.http_scheme() + url)
 
         parsed_url = compat_urlparse.urlparse(url)
         if not parsed_url.scheme:
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index c4776bbf3..e55b1a202 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -363,7 +363,4 @@ class LivestreamShortenerIE(InfoExtractor):
         id = mobj.group('id')
         webpage = self._download_webpage(url, id)
 
-        return {
-            '_type': 'url',
-            'url': self._og_search_url(webpage),
-        }
+        return self.url_result(self._og_search_url(webpage))
diff --git a/youtube_dl/extractor/savefrom.py b/youtube_dl/extractor/savefrom.py
index 30f9cf824..21e44b69a 100644
--- a/youtube_dl/extractor/savefrom.py
+++ b/youtube_dl/extractor/savefrom.py
@@ -30,8 +30,5 @@ class SaveFromIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = os.path.splitext(url.split('/')[-1])[0]
-        return {
-            '_type': 'url',
-            'id': video_id,
-            'url': mobj.group('url'),
-        }
+
+        return self.url_result(mobj.group('url'), video_id=video_id)
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index f9b6aa48f..d3e4205f5 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -203,10 +203,8 @@ class TEDIE(InfoExtractor):
                 ext_url = None
                 if service.lower() == 'youtube':
                     ext_url = external.get('code')
-                return {
-                    '_type': 'url',
-                    'url': ext_url or external['uri'],
-                }
+
+                return self.url_result(ext_url or external['uri'])
 
         resources_ = player_talk.get('resources') or talk_info.get('resources')
 
diff --git a/youtube_dl/extractor/testurl.py b/youtube_dl/extractor/testurl.py
index 46918adb0..84a14a0bd 100644
--- a/youtube_dl/extractor/testurl.py
+++ b/youtube_dl/extractor/testurl.py
@@ -61,8 +61,4 @@ class TestURLIE(InfoExtractor):
 
         self.to_screen('Test URL: %s' % tc['url'])
 
-        return {
-            '_type': 'url',
-            'url': tc['url'],
-            'id': video_id,
-        }
+        return self.url_result(tc['url'], video_id=video_id)
diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 3dab9145b..ea234e3c5 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -40,11 +40,7 @@ class WimpIE(InfoExtractor):
              r'data-id=["\']([0-9A-Za-z_-]{11})'),
             webpage, 'video URL', default=None)
         if youtube_id:
-            return {
-                '_type': 'url',
-                'url': youtube_id,
-                'ie_key': YoutubeIE.ie_key(),
-            }
+            return self.url_result(youtube_id, YoutubeIE.ie_key())
 
         info_dict = self._extract_jwplayer_data(
             webpage, video_id, require_title=False)

From 751e051557372ab3fa41d839e7e6a149aa2d04b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Jan 2019 17:26:15 +0700
Subject: [PATCH 280/510] [packtpub] Add support for subscription.packtpub.com
 (closes #18718)

---
 youtube_dl/extractor/packtpub.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
index 56a2a1083..1324137df 100644
--- a/youtube_dl/extractor/packtpub.py
+++ b/youtube_dl/extractor/packtpub.py
@@ -24,9 +24,9 @@ class PacktPubBaseIE(InfoExtractor):
 
 
 class PacktPubIE(PacktPubBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?packtpub\.com/mapt/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
         'info_dict': {
@@ -37,7 +37,10 @@ class PacktPubIE(PacktPubBaseIE):
             'timestamp': 1490918400,
             'upload_date': '20170331',
         },
-    }
+    }, {
+        'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
+        'only_matching': True,
+    }]
     _NETRC_MACHINE = 'packtpub'
     _TOKEN = None
 
@@ -110,15 +113,18 @@ class PacktPubIE(PacktPubBaseIE):
 
 
 class PacktPubCourseIE(PacktPubBaseIE):
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?packtpub\.com/mapt/video/[^/]+/(?P<id>\d+))'
-    _TEST = {
+    _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
+    _TESTS = [{
         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
         'info_dict': {
             'id': '9781787122215',
             'title': 'Learn Nodejs by building 12 projects [Video]',
         },
         'playlist_count': 90,
-    }
+    }, {
+        'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
+        'only_matching': True,
+    }]
 
     @classmethod
     def suitable(cls, url):

From 2122d7151deca9c7047426320cd73152cb907afa Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Wed, 2 Jan 2019 17:46:07 +0100
Subject: [PATCH 281/510] [discovery] Use geo verification headers

---
 youtube_dl/extractor/discovery.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 44fbc41bb..b70c307a7 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -94,11 +94,12 @@ class DiscoveryIE(DiscoveryGoBaseIE):
                 })['access_token']
 
         try:
+            headers = self.geo_verification_headers()
+            headers['Authorization'] = 'Bearer ' + access_token
+
             stream = self._download_json(
                 'https://api.discovery.com/v1/streaming/video/' + video_id,
-                display_id, headers={
-                    'Authorization': 'Bearer ' + access_token,
-                })
+                display_id, headers=headers)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
                 e_description = self._parse_json(

From aeb72b3a413c8a9d11e3c002b8ab2612b9075a56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Jan 2019 23:51:23 +0700
Subject: [PATCH 282/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 72bfe2110..1a28d6be0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Extractors
+* [discovery] Use geo verification headers (#17838)
++ [packtpub] Add support for subscription.packtpub.com (#18718)
+* [yourporn] Fix extraction (#18583)
++ [acast:channel] Add support for play.acast.com (#18587)
++ [extractors] Add missing age limits (#18621)
++ [rmcdecouverte] Add support for live stream
+* [rmcdecouverte] Bypass geo restriction
+* [rmcdecouverte] Update URL regular expression (#18595, 18697)
+* [manyvids] Fix extraction (#18604, #18614)
+* [bitchute] Fix extraction (#18567)
+
+
 version 2018.12.31
 
 Extractors

From d7c3af7a72a1e6bd7f93321cce4daf2a13ebdf12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Jan 2019 23:52:54 +0700
Subject: [PATCH 283/510] release 2019.01.02

---
 .github/ISSUE_TEMPLATE.md |  6 ++---
 CONTRIBUTING.md           | 47 ++++++++++++++++++++++++++++++++++++---
 ChangeLog                 |  2 +-
 youtube_dl/version.py     |  2 +-
 4 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b2fe6ee6a..3986bbc8e 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.12.31*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.12.31**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.12.31
+[debug] youtube-dl version 2019.01.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 24617264c..a71b045d0 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -261,11 +261,33 @@ title = meta.get('title') or self._og_search_title(webpage)
 
 This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
 
-### Make regular expressions flexible
+### Regular expressions
 
-When using regular expressions try to write them fuzzy and flexible.
+#### Don't capture groups you don't use
+
+Capturing group must be an indication that it's used somewhere in the code. Any group that is not used must be non capturing.
+
+##### Example
+
+Don't capture id attribute name here since you can't use it for anything anyway.
+
+Correct:
+
+```python
+r'(?:id|ID)=(?P<id>\d+)'
+```
+
+Incorrect:
+```python
+r'(id|ID)=(?P<id>\d+)'
+```
+
+
+#### Make regular expressions relaxed and flexible
+
+When using regular expressions try to write them fuzzy, relaxed and flexible, skipping insignificant parts that are more likely to change, allowing both single and double quotes for quoted values and so on.
  
-#### Example
+##### Example
 
 Say you need to extract `title` from the following HTML code:
 
@@ -298,6 +320,25 @@ title = self._search_regex(
     webpage, 'title', group='title')
 ```
 
+### Long lines policy
+
+There is a soft limit to keep lines of code under 80 characters long. This means it should be respected if possible and if it does not make readability and code maintenance worse.
+
+For example, you should **never** split long string literals like URLs or some other often copied entities over multiple lines to fit this limit:
+
+Correct:
+
+```python
+'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
+```
+
+Incorrect:
+
+```python
+'https://www.youtube.com/watch?v=FqZTN594JQw&list='
+'PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
+```
+
 ### Use safe conversion functions
 
 Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
diff --git a/ChangeLog b/ChangeLog
index 1a28d6be0..41d190a72 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.02
 
 Extractors
 * [discovery] Use geo verification headers (#17838)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 87de5848d..07a444706 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.12.31'
+__version__ = '2019.01.02'

From c87f65e43dea0f9edf1b5ed8979e274902fb60a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Jan 2019 22:21:53 +0700
Subject: [PATCH 284/510] [carambatv:page] Fix extraction (closes #18739)

---
 youtube_dl/extractor/carambatv.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/carambatv.py b/youtube_dl/extractor/carambatv.py
index 9ba909a91..b57b86af7 100644
--- a/youtube_dl/extractor/carambatv.py
+++ b/youtube_dl/extractor/carambatv.py
@@ -82,6 +82,12 @@ class CarambaTVPageIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         videomore_url = VideomoreIE._extract_url(webpage)
+        if not videomore_url:
+            videomore_id = self._search_regex(
+                r'getVMCode\s*\(\s*["\']?(\d+)', webpage, 'videomore id',
+                default=None)
+            if videomore_id:
+                videomore_url = 'videomore:%s' % videomore_id
         if videomore_url:
             title = self._og_search_title(webpage)
             return {

From de0359c0af3667605464212e66ba4048b6ba093b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Jan 2019 03:40:41 +0700
Subject: [PATCH 285/510] [tvnow] Fix and rework extractors, prepare for a
 switch to the new API (closes #17245, closes #18499)

---
 youtube_dl/extractor/extractors.py |   4 +-
 youtube_dl/extractor/tvnow.py      | 378 +++++++++++++++++++++--------
 2 files changed, 283 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d72f52e36..3b1dfc451 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1193,7 +1193,9 @@ from .tvnet import TVNetIE
 from .tvnoe import TVNoeIE
 from .tvnow import (
     TVNowIE,
-    TVNowListIE,
+    TVNowNewIE,
+    TVNowSeasonIE,
+    TVNowAnnualIE,
     TVNowShowIE,
 )
 from .tvp import (
diff --git a/youtube_dl/extractor/tvnow.py b/youtube_dl/extractor/tvnow.py
index 60937616f..3c6a60c39 100644
--- a/youtube_dl/extractor/tvnow.py
+++ b/youtube_dl/extractor/tvnow.py
@@ -10,8 +10,9 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     parse_duration,
-    try_get,
+    str_or_none,
     update_url_query,
+    urljoin,
 )
 
 
@@ -24,8 +25,7 @@ class TVNowBaseIE(InfoExtractor):
 
     def _call_api(self, path, video_id, query):
         return self._download_json(
-            'https://api.tvnow.de/v3/' + path,
-            video_id, query=query)
+            'https://api.tvnow.de/v3/' + path, video_id, query=query)
 
     def _extract_video(self, info, display_id):
         video_id = compat_str(info['id'])
@@ -108,6 +108,11 @@ class TVNowIE(TVNowBaseIE):
                         (?!(?:list|jahr)(?:/|$))(?P<id>[^/?\#&]+)
                     '''
 
+    @classmethod
+    def suitable(cls, url):
+        return (False if TVNowNewIE.suitable(url) or TVNowSeasonIE.suitable(url) or TVNowAnnualIE.suitable(url) or TVNowShowIE.suitable(url)
+                else super(TVNowIE, cls).suitable(url))
+
     _TESTS = [{
         'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/der-neue-porsche-911-gt-3/player',
         'info_dict': {
@@ -116,7 +121,6 @@ class TVNowIE(TVNowBaseIE):
             'ext': 'mp4',
             'title': 'Der neue Porsche 911 GT 3',
             'description': 'md5:6143220c661f9b0aae73b245e5d898bb',
-            'thumbnail': r're:^https?://.*\.jpg$',
             'timestamp': 1495994400,
             'upload_date': '20170528',
             'duration': 5283,
@@ -161,136 +165,314 @@ class TVNowIE(TVNowBaseIE):
         info = self._call_api(
             'movies/' + display_id, display_id, query={
                 'fields': ','.join(self._VIDEO_FIELDS),
-                'station': mobj.group(1),
             })
 
         return self._extract_video(info, display_id)
 
 
-class TVNowListBaseIE(TVNowBaseIE):
-    _SHOW_VALID_URL = r'''(?x)
-                    (?P<base_url>
-                        https?://
-                            (?:www\.)?tvnow\.(?:de|at|ch)/[^/]+/
-                            (?P<show_id>[^/]+)
-                    )
+class TVNowNewIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?P<base_url>https?://
+                        (?:www\.)?tvnow\.(?:de|at|ch)/
+                        (?:shows|serien))/
+                        (?P<show>[^/]+)-\d+/
+                        [^/]+/
+                        episode-\d+-(?P<episode>[^/?$&]+)-(?P<id>\d+)
                     '''
 
-    def _extract_list_info(self, display_id, show_id):
-        fields = list(self._SHOW_FIELDS)
-        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
-        fields.extend(
-            'formatTabs.formatTabPages.container.movies.%s' % field
-            for field in self._VIDEO_FIELDS)
-        return self._call_api(
-            'formats/seo', display_id, query={
-                'fields': ','.join(fields),
-                'name': show_id + '.php'
-            })
-
-
-class TVNowListIE(TVNowListBaseIE):
-    _VALID_URL = r'%s/(?:list|jahr)/(?P<id>[^?\#&]+)' % TVNowListBaseIE._SHOW_VALID_URL
-
-    _SHOW_FIELDS = ('title', )
-    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
-    _VIDEO_FIELDS = ('id', 'headline', 'seoUrl', )
-
     _TESTS = [{
-        'url': 'https://www.tvnow.de/rtl/30-minuten-deutschland/list/aktuell',
-        'info_dict': {
-            'id': '28296',
-            'title': '30 Minuten Deutschland - Aktuell',
-        },
-        'playlist_mincount': 1,
-    }, {
-        'url': 'https://www.tvnow.de/vox/ab-ins-beet/list/staffel-14',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/jahr/2018/3',
+        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
         'only_matching': True,
     }]
 
-    @classmethod
-    def suitable(cls, url):
-        return (False if TVNowIE.suitable(url)
-                else super(TVNowListIE, cls).suitable(url))
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        base_url = re.sub(r'(?:shows|serien)', '_', mobj.group('base_url'))
+        show, episode = mobj.group('show', 'episode')
+        return self.url_result(
+            # Rewrite new URLs to the old format and use extraction via old API
+            # at api.tvnow.de as a loophole for bypassing premium content checks
+            '%s/%s/%s' % (base_url, show, episode),
+            ie=TVNowIE.ie_key(), video_id=mobj.group('id'))
+
+
+class TVNowNewBaseIE(InfoExtractor):
+    def _call_api(self, path, video_id, query={}):
+        result = self._download_json(
+            'https://apigw.tvnow.de/module/' + path, video_id, query=query)
+        error = result.get('error')
+        if error:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error), expected=True)
+        return result
+
+
+"""
+TODO: new apigw.tvnow.de based version of TVNowIE. Replace old TVNowIE with it
+when api.tvnow.de is shut down. This version can't bypass premium checks though.
+class TVNowIE(TVNowNewBaseIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?tvnow\.(?:de|at|ch)/
+                        (?:shows|serien)/[^/]+/
+                        (?:[^/]+/)+
+                        (?P<display_id>[^/?$&]+)-(?P<id>\d+)
+                    '''
+
+    _TESTS = [{
+        # episode with annual navigation
+        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
+        'info_dict': {
+            'id': '331082',
+            'display_id': 'grip-das-motormagazin/der-neue-porsche-911-gt-3',
+            'ext': 'mp4',
+            'title': 'Der neue Porsche 911 GT 3',
+            'description': 'md5:6143220c661f9b0aae73b245e5d898bb',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1495994400,
+            'upload_date': '20170528',
+            'duration': 5283,
+            'series': 'GRIP - Das Motormagazin',
+            'season_number': 14,
+            'episode_number': 405,
+            'episode': 'Der neue Porsche 911 GT 3',
+        },
+    }, {
+        # rtl2, episode with season navigation
+        'url': 'https://www.tvnow.de/shows/armes-deutschland-11471/staffel-3/episode-14-bernd-steht-seit-der-trennung-von-seiner-frau-allein-da-526124',
+        'only_matching': True,
+    }, {
+        # rtlnitro
+        'url': 'https://www.tvnow.de/serien/alarm-fuer-cobra-11-die-autobahnpolizei-1815/staffel-13/episode-5-auf-eigene-faust-pilot-366822',
+        'only_matching': True,
+    }, {
+        # superrtl
+        'url': 'https://www.tvnow.de/shows/die-lustigsten-schlamassel-der-welt-1221/staffel-2/episode-14-u-a-ketchup-effekt-364120',
+        'only_matching': True,
+    }, {
+        # ntv
+        'url': 'https://www.tvnow.de/shows/startup-news-10674/staffel-2/episode-39-goetter-in-weiss-387630',
+        'only_matching': True,
+    }, {
+        # vox
+        'url': 'https://www.tvnow.de/shows/auto-mobil-174/2017-11/episode-46-neues-vom-automobilmarkt-2017-11-19-17-00-00-380072',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
+        'only_matching': True,
+    }]
+
+    def _extract_video(self, info, url, display_id):
+        config = info['config']
+        source = config['source']
+
+        video_id = compat_str(info.get('id') or source['videoId'])
+        title = source['title'].strip()
+
+        paths = []
+        for manifest_url in (info.get('manifest') or {}).values():
+            if not manifest_url:
+                continue
+            manifest_url = update_url_query(manifest_url, {'filter': ''})
+            path = self._search_regex(r'https?://[^/]+/(.+?)\.ism/', manifest_url, 'path')
+            if path in paths:
+                continue
+            paths.append(path)
+
+            def url_repl(proto, suffix):
+                return re.sub(
+                    r'(?:hls|dash|hss)([.-])', proto + r'\1', re.sub(
+                        r'\.ism/(?:[^.]*\.(?:m3u8|mpd)|[Mm]anifest)',
+                        '.ism/' + suffix, manifest_url))
+
+            formats = self._extract_mpd_formats(
+                url_repl('dash', '.mpd'), video_id,
+                mpd_id='dash', fatal=False)
+            formats.extend(self._extract_ism_formats(
+                url_repl('hss', 'Manifest'),
+                video_id, ism_id='mss', fatal=False))
+            formats.extend(self._extract_m3u8_formats(
+                url_repl('hls', '.m3u8'), video_id, 'mp4',
+                'm3u8_native', m3u8_id='hls', fatal=False))
+            if formats:
+                break
+        else:
+            if try_get(info, lambda x: x['rights']['isDrm']):
+                raise ExtractorError(
+                    'Video %s is DRM protected' % video_id, expected=True)
+            if try_get(config, lambda x: x['boards']['geoBlocking']['block']):
+                raise self.raise_geo_restricted()
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
+        self._sort_formats(formats)
+
+        description = source.get('description')
+        thumbnail = url_or_none(source.get('poster'))
+        timestamp = unified_timestamp(source.get('previewStart'))
+        duration = parse_duration(source.get('length'))
+
+        series = source.get('format')
+        season_number = int_or_none(self._search_regex(
+            r'staffel-(\d+)', url, 'season number', default=None))
+        episode_number = int_or_none(self._search_regex(
+            r'episode-(\d+)', url, 'episode number', default=None))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'series': series,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'episode': title,
+            'formats': formats,
+        }
 
     def _real_extract(self, url):
-        base_url, show_id, season_id = re.match(self._VALID_URL, url).groups()
+        display_id, video_id = re.match(self._VALID_URL, url).groups()
+        info = self._call_api('player/' + video_id, video_id)
+        return self._extract_video(info, video_id, display_id)
+"""
 
-        list_info = self._extract_list_info(season_id, show_id)
 
-        season = next(
-            season for season in list_info['formatTabs']['items']
-            if season.get('seoheadline') == season_id)
+class TVNowListBaseIE(TVNowNewBaseIE):
+    _SHOW_VALID_URL = r'''(?x)
+                    (?P<base_url>
+                        https?://
+                            (?:www\.)?tvnow\.(?:de|at|ch)/(?:shows|serien)/
+                            [^/?#&]+-(?P<show_id>\d+)
+                    )
+                    '''
 
-        title = list_info.get('title')
-        headline = season.get('headline')
-        if title and headline:
-            title = '%s - %s' % (title, headline)
-        else:
-            title = headline or title
+    @classmethod
+    def suitable(cls, url):
+        return (False if TVNowNewIE.suitable(url)
+                else super(TVNowListBaseIE, cls).suitable(url))
+
+    def _extract_items(self, url, show_id, list_id, query):
+        items = self._call_api(
+            'teaserrow/format/episode/' + show_id, list_id,
+            query=query)['items']
 
         entries = []
-        for container in season['formatTabPages']['items']:
-            items = try_get(
-                container, lambda x: x['container']['movies']['items'],
-                list) or []
-            for info in items:
-                seo_url = info.get('seoUrl')
-                if not seo_url:
-                    continue
-                video_id = info.get('id')
-                entries.append(self.url_result(
-                    '%s/%s/player' % (base_url, seo_url), TVNowIE.ie_key(),
-                    compat_str(video_id) if video_id else None))
+        for item in items:
+            if not isinstance(item, dict):
+                continue
+            item_url = urljoin(url, item.get('url'))
+            if not item_url:
+                continue
+            video_id = str_or_none(item.get('id') or item.get('videoId'))
+            item_title = item.get('subheadline') or item.get('text')
+            entries.append(self.url_result(
+                item_url, ie=TVNowNewIE.ie_key(), video_id=video_id,
+                video_title=item_title))
 
-        return self.playlist_result(
-            entries, compat_str(season.get('id') or season_id), title)
+        return self.playlist_result(entries, '%s/%s' % (show_id, list_id))
+
+
+class TVNowSeasonIE(TVNowListBaseIE):
+    _VALID_URL = r'%s/staffel-(?P<id>\d+)' % TVNowListBaseIE._SHOW_VALID_URL
+    _TESTS = [{
+        'url': 'https://www.tvnow.de/serien/alarm-fuer-cobra-11-die-autobahnpolizei-1815/staffel-13',
+        'info_dict': {
+            'id': '1815/13',
+        },
+        'playlist_mincount': 22,
+    }]
+
+    def _real_extract(self, url):
+        _, show_id, season_id = re.match(self._VALID_URL, url).groups()
+        return self._extract_items(
+            url, show_id, season_id, {'season': season_id})
+
+
+class TVNowAnnualIE(TVNowListBaseIE):
+    _VALID_URL = r'%s/(?P<year>\d{4})-(?P<month>\d{2})' % TVNowListBaseIE._SHOW_VALID_URL
+    _TESTS = [{
+        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05',
+        'info_dict': {
+            'id': '1669/2017-05',
+        },
+        'playlist_mincount': 2,
+    }]
+
+    def _real_extract(self, url):
+        _, show_id, year, month = re.match(self._VALID_URL, url).groups()
+        return self._extract_items(
+            url, show_id, '%s-%s' % (year, month), {
+                'year': int(year),
+                'month': int(month),
+            })
 
 
 class TVNowShowIE(TVNowListBaseIE):
     _VALID_URL = TVNowListBaseIE._SHOW_VALID_URL
-
-    _SHOW_FIELDS = ('id', 'title', )
-    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
-    _VIDEO_FIELDS = ()
-
     _TESTS = [{
-        'url': 'https://www.tvnow.at/vox/ab-ins-beet',
+        # annual navigationType
+        'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669',
         'info_dict': {
-            'id': 'ab-ins-beet',
-            'title': 'Ab ins Beet!',
+            'id': '1669',
         },
-        'playlist_mincount': 7,
+        'playlist_mincount': 73,
     }, {
-        'url': 'https://www.tvnow.at/vox/ab-ins-beet/list',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/jahr/',
-        'only_matching': True,
+        # season navigationType
+        'url': 'https://www.tvnow.de/shows/armes-deutschland-11471',
+        'info_dict': {
+            'id': '11471',
+        },
+        'playlist_mincount': 3,
     }]
 
     @classmethod
     def suitable(cls, url):
-        return (False if TVNowIE.suitable(url) or TVNowListIE.suitable(url)
+        return (False if TVNowNewIE.suitable(url) or TVNowSeasonIE.suitable(url) or TVNowAnnualIE.suitable(url)
                 else super(TVNowShowIE, cls).suitable(url))
 
     def _real_extract(self, url):
         base_url, show_id = re.match(self._VALID_URL, url).groups()
 
-        list_info = self._extract_list_info(show_id, show_id)
+        result = self._call_api(
+            'teaserrow/format/navigation/' + show_id, show_id)
+
+        items = result['items']
 
         entries = []
-        for season_info in list_info['formatTabs']['items']:
-            season_url = season_info.get('seoheadline')
-            if not season_url:
-                continue
-            season_id = season_info.get('id')
-            entries.append(self.url_result(
-                '%s/list/%s' % (base_url, season_url), TVNowListIE.ie_key(),
-                compat_str(season_id) if season_id else None,
-                season_info.get('headline')))
+        navigation = result.get('navigationType')
+        if navigation == 'annual':
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                year = int_or_none(item.get('year'))
+                if year is None:
+                    continue
+                months = item.get('months')
+                if not isinstance(months, list):
+                    continue
+                for month_dict in months:
+                    if not isinstance(month_dict, dict) or not month_dict:
+                        continue
+                    month_number = int_or_none(list(month_dict.keys())[0])
+                    if month_number is None:
+                        continue
+                    entries.append(self.url_result(
+                        '%s/%04d-%02d' % (base_url, year, month_number),
+                        ie=TVNowAnnualIE.ie_key()))
+        elif navigation == 'season':
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                season_number = int_or_none(item.get('season'))
+                if season_number is None:
+                    continue
+                entries.append(self.url_result(
+                    '%s/staffel-%d' % (base_url, season_number),
+                    ie=TVNowSeasonIE.ie_key()))
+        else:
+            raise ExtractorError('Unknown navigationType')
 
-        return self.playlist_result(entries, show_id, list_info.get('title'))
+        return self.playlist_result(entries, show_id)

From b7acc83550598b9facf1b31d3d9a1c2823b29cbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Jan 2019 00:55:39 +0700
Subject: [PATCH 286/510] [utils] Add language codes replaced in 1989 revision
 of ISO 639 to ISO639Utils (closes #18765)

---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 62e769fd5..64b524d2f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2968,6 +2968,7 @@ class ISO639Utils(object):
         'gv': 'glv',
         'ha': 'hau',
         'he': 'heb',
+        'iw': 'heb',  # Replaced by he in 1989 revision
         'hi': 'hin',
         'ho': 'hmo',
         'hr': 'hrv',
@@ -2977,6 +2978,7 @@ class ISO639Utils(object):
         'hz': 'her',
         'ia': 'ina',
         'id': 'ind',
+        'in': 'ind',  # Replaced by id in 1989 revision
         'ie': 'ile',
         'ig': 'ibo',
         'ii': 'iii',
@@ -3091,6 +3093,7 @@ class ISO639Utils(object):
         'wo': 'wol',
         'xh': 'xho',
         'yi': 'yid',
+        'ji': 'yid',  # Replaced by he yi 1989 revision
         'yo': 'yor',
         'za': 'zha',
         'zh': 'zho',

From 04fb6928da5e34fb3c7eaf50592a141400e00487 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Jan 2019 00:57:24 +0700
Subject: [PATCH 287/510] [postprocessor/ffmpeg] Embed subtitles with
 non-standard language codes (refs #18765)

---
 youtube_dl/postprocessor/ffmpeg.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 757b496a1..efcd39d57 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -384,9 +384,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             opts += ['-c:s', 'mov_text']
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1)])
-            lang_code = ISO639Utils.short2long(lang)
-            if lang_code is not None:
-                opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
+            lang_code = ISO639Utils.short2long(lang) or lang
+            opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
 
         temp_filename = prepend_extension(filename, 'temp')
         self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename)

From e9a50fba8603128dcb81fa1b59206a0cb106d540 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Jan 2019 01:02:34 +0700
Subject: [PATCH 288/510] [utils] Fix typo

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 64b524d2f..d2d3c1a9f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3093,7 +3093,7 @@ class ISO639Utils(object):
         'wo': 'wol',
         'xh': 'xho',
         'yi': 'yid',
-        'ji': 'yid',  # Replaced by he yi 1989 revision
+        'ji': 'yid',  # Replaced by yi in 1989 revision
         'yo': 'yor',
         'za': 'zha',
         'zh': 'zho',

From bcc334a3c66f2465cd63be284e389cc4c7a78203 Mon Sep 17 00:00:00 2001
From: 4rensiker <4rensics@gmx.de>
Date: Tue, 8 Jan 2019 02:44:42 +0100
Subject: [PATCH 289/510] [dtube] Fix extraction (closes #18741)

---
 youtube_dl/extractor/dtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dtube.py b/youtube_dl/extractor/dtube.py
index 5887887e1..20190c9cc 100644
--- a/youtube_dl/extractor/dtube.py
+++ b/youtube_dl/extractor/dtube.py
@@ -48,7 +48,7 @@ class DTubeIE(InfoExtractor):
         def canonical_url(h):
             if not h:
                 return None
-            return 'https://ipfs.io/ipfs/' + h
+            return 'https://video.dtube.top/ipfs/' + h
 
         formats = []
         for q in ('240', '480', '720', '1080', ''):

From 0266854f63baaa9669b30ee49c5e7e9f353970dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Jan 2019 08:46:34 +0700
Subject: [PATCH 290/510] [dtube] Update test

---
 youtube_dl/extractor/dtube.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/dtube.py b/youtube_dl/extractor/dtube.py
index 20190c9cc..114d2dbe3 100644
--- a/youtube_dl/extractor/dtube.py
+++ b/youtube_dl/extractor/dtube.py
@@ -15,16 +15,16 @@ from ..utils import (
 class DTubeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?d\.tube/(?:#!/)?v/(?P<uploader_id>[0-9a-z.-]+)/(?P<id>[0-9a-z]{8})'
     _TEST = {
-        'url': 'https://d.tube/#!/v/benswann/zqd630em',
-        'md5': 'a03eaa186618ffa7a3145945543a251e',
+        'url': 'https://d.tube/#!/v/broncnutz/x380jtr1',
+        'md5': '9f29088fa08d699a7565ee983f56a06e',
         'info_dict': {
-            'id': 'zqd630em',
+            'id': 'x380jtr1',
             'ext': 'mp4',
-            'title': 'Reality Check: FDA\'s Disinformation Campaign on Kratom',
-            'description': 'md5:700d164e066b87f9eac057949e4227c2',
-            'uploader_id': 'benswann',
-            'upload_date': '20180222',
-            'timestamp': 1519328958,
+            'title': 'Lefty 3-Rings is Back Baby!! NCAA Picks',
+            'description': 'md5:60be222088183be3a42f196f34235776',
+            'uploader_id': 'broncnutz',
+            'upload_date': '20190107',
+            'timestamp': 1546854054,
         },
         'params': {
             'format': '480p',

From 8cb5c2181ac039e92a20247f266f48cee2411748 Mon Sep 17 00:00:00 2001
From: Awal Garg <awalgarg@gmail.com>
Date: Mon, 7 Jan 2019 17:41:12 +0530
Subject: [PATCH 291/510] [hungama] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/hungama.py    | 32 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/hungama.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3b1dfc451..5ff34216f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -469,6 +469,7 @@ from .hrti import (
 )
 from .huajiao import HuajiaoIE
 from .huffpost import HuffPostIE
+from .hungama import HungamaIE
 from .hypem import HypemIE
 from .iconosquare import IconosquareIE
 from .ign import (
diff --git a/youtube_dl/extractor/hungama.py b/youtube_dl/extractor/hungama.py
new file mode 100644
index 000000000..e51440299
--- /dev/null
+++ b/youtube_dl/extractor/hungama.py
@@ -0,0 +1,32 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class HungamaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)hungama\.com/song/[\w\d-]+/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.hungama.com/song/kitni-haseen-zindagi/2931166/',
+        'md5': '396fa7e8e7e67aa25da0edc4cac9b785',
+        'info_dict': {
+            'id': '2931166',
+            'ext': 'mp4',
+            'title': 'Kitni Haseen Zindagi',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        player_data = self._download_json('https://www.hungama.com/audio-player-data/track/%s?_country=IN' % video_id, video_id)[0]
+        title = player_data.get('song_name') or self._og_search_title(webpage)
+        track_data = self._download_json(player_data['file'], video_id)
+        media_url = track_data['response']['media_url']
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': self._extract_m3u8_formats(media_url, video_id, ext='mp4'),
+        }

From 06b4b90c70636d8044b7a2f588503cadf8b202d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Jan 2019 09:09:49 +0700
Subject: [PATCH 292/510] [hungama] Fix code and extract more metadata (closes
 #18771)

---
 youtube_dl/extractor/hungama.py | 41 +++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/hungama.py b/youtube_dl/extractor/hungama.py
index e51440299..d9ad1281a 100644
--- a/youtube_dl/extractor/hungama.py
+++ b/youtube_dl/extractor/hungama.py
@@ -2,31 +2,54 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class HungamaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)hungama\.com/song/[\w\d-]+/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?hungama\.com/song/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://www.hungama.com/song/kitni-haseen-zindagi/2931166/',
-        'md5': '396fa7e8e7e67aa25da0edc4cac9b785',
+        'md5': 'a845a6d1ebd08d80c1035126d49bd6a0',
         'info_dict': {
             'id': '2931166',
             'ext': 'mp4',
-            'title': 'Kitni Haseen Zindagi',
+            'title': 'Lucky Ali - Kitni Haseen Zindagi',
+            'track': 'Kitni Haseen Zindagi',
+            'artist': 'Lucky Ali',
+            'album': 'Aks',
+            'release_year': 2000,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        player_data = self._download_json('https://www.hungama.com/audio-player-data/track/%s?_country=IN' % video_id, video_id)[0]
-        title = player_data.get('song_name') or self._og_search_title(webpage)
-        track_data = self._download_json(player_data['file'], video_id)
-        media_url = track_data['response']['media_url']
+        data = self._download_json(
+            'https://www.hungama.com/audio-player-data/track/%s' % video_id,
+            video_id, query={'_country': 'IN'})[0]
+
+        track = data['song_name']
+        artist = data.get('singer_name')
+
+        m3u8_url = self._download_json(
+            data.get('file') or data['preview_link'],
+            video_id)['response']['media_url']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+        self._sort_formats(formats)
+
+        title = '%s - %s' % (artist, track) if artist else track
+        thumbnail = data.get('img_src') or data.get('album_image')
 
         return {
             'id': video_id,
             'title': title,
-            'formats': self._extract_m3u8_formats(media_url, video_id, ext='mp4'),
+            'thumbnail': thumbnail,
+            'track': track,
+            'artist': artist,
+            'album': data.get('album_name'),
+            'release_year': int_or_none(data.get('date')),
+            'formats': formats,
         }

From 391256dc0ee5e6ac8d3022455de207b8b02a5b10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Jan 2019 10:02:00 +0700
Subject: [PATCH 293/510] [extractor/common] Add support for movies in _json_ld

---
 youtube_dl/extractor/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e5f8136fc..f507400cc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1250,6 +1250,13 @@ class InfoExtractor(object):
                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
                     if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
                         info['series'] = unescapeHTML(part_of_series.get('name'))
+                elif item_type == 'Movie':
+                    info.update({
+                        'title': unescapeHTML(e.get('name')),
+                        'description': unescapeHTML(e.get('description')),
+                        'duration': parse_duration(e.get('duration')),
+                        'timestamp': unified_timestamp(e.get('dateCreated')),
+                    })
                 elif item_type in ('Article', 'NewsArticle'):
                     info.update({
                         'timestamp': parse_iso8601(e.get('datePublished')),

From 440863ade10c70bec272c1ffe17b63c877e46ca7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Jan 2019 10:02:49 +0700
Subject: [PATCH 294/510] [extractor/common] Use episode name as title in
 _json_ld

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f507400cc..9e7febcad 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1239,11 +1239,14 @@ class InfoExtractor(object):
                 if expected_type is not None and expected_type != item_type:
                     return info
                 if item_type in ('TVEpisode', 'Episode'):
+                    episode_name = unescapeHTML(e.get('name'))
                     info.update({
-                        'episode': unescapeHTML(e.get('name')),
+                        'episode': episode_name,
                         'episode_number': int_or_none(e.get('episodeNumber')),
                         'description': unescapeHTML(e.get('description')),
                     })
+                    if not info.get('title') and episode_name:
+                        info['title'] = episode_name
                     part_of_season = e.get('partOfSeason')
                     if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
                         info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))

From 2543938bbe393ceef8dca6a69b441d54df099107 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Jan 2019 10:03:44 +0700
Subject: [PATCH 295/510] [hungama] Add support for videos (closes #17402)

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/hungama.py    | 78 +++++++++++++++++++++++++++---
 2 files changed, 74 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5ff34216f..ddeb70284 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -469,7 +469,10 @@ from .hrti import (
 )
 from .huajiao import HuajiaoIE
 from .huffpost import HuffPostIE
-from .hungama import HungamaIE
+from .hungama import (
+    HungamaIE,
+    HungamaSongIE,
+)
 from .hypem import HypemIE
 from .iconosquare import IconosquareIE
 from .ign import (
diff --git a/youtube_dl/extractor/hungama.py b/youtube_dl/extractor/hungama.py
index d9ad1281a..3fdaac5b6 100644
--- a/youtube_dl/extractor/hungama.py
+++ b/youtube_dl/extractor/hungama.py
@@ -2,11 +2,73 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    urlencode_postdata,
+)
 
 
 class HungamaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hungama\.com/song/[^/]+/(?P<id>[0-9]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?hungama\.com/
+                        (?:
+                            (?:video|movie)/[^/]+/|
+                            tv-show/(?:[^/]+/){2}\d+/episode/[^/]+/
+                        )
+                        (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'http://www.hungama.com/video/krishna-chants/39349649/',
+        'md5': 'a845a6d1ebd08d80c1035126d49bd6a0',
+        'info_dict': {
+            'id': '2931166',
+            'ext': 'mp4',
+            'title': 'Lucky Ali - Kitni Haseen Zindagi',
+            'track': 'Kitni Haseen Zindagi',
+            'artist': 'Lucky Ali',
+            'album': 'Aks',
+            'release_year': 2000,
+        }
+    }, {
+        'url': 'https://www.hungama.com/movie/kahaani-2/44129919/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.hungama.com/tv-show/padded-ki-pushup/season-1/44139461/episode/ep-02-training-sasu-pathlaag-karing/44139503/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        info = self._search_json_ld(webpage, video_id)
+
+        m3u8_url = self._download_json(
+            'https://www.hungama.com/index.php', video_id,
+            data=urlencode_postdata({'content_id': video_id}), headers={
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                'X-Requested-With': 'XMLHttpRequest',
+            }, query={
+                'c': 'common',
+                'm': 'get_video_mdn_url',
+            })['stream_url']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+        self._sort_formats(formats)
+
+        info.update({
+            'id': video_id,
+            'formats': formats,
+        })
+        return info
+
+
+class HungamaSongIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hungama\.com/song/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.hungama.com/song/kitni-haseen-zindagi/2931166/',
         'md5': 'a845a6d1ebd08d80c1035126d49bd6a0',
@@ -22,21 +84,21 @@ class HungamaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        audio_id = self._match_id(url)
 
         data = self._download_json(
-            'https://www.hungama.com/audio-player-data/track/%s' % video_id,
-            video_id, query={'_country': 'IN'})[0]
+            'https://www.hungama.com/audio-player-data/track/%s' % audio_id,
+            audio_id, query={'_country': 'IN'})[0]
 
         track = data['song_name']
         artist = data.get('singer_name')
 
         m3u8_url = self._download_json(
             data.get('file') or data['preview_link'],
-            video_id)['response']['media_url']
+            audio_id)['response']['media_url']
 
         formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+            m3u8_url, audio_id, ext='mp4', entry_protocol='m3u8_native',
             m3u8_id='hls')
         self._sort_formats(formats)
 
@@ -44,7 +106,7 @@ class HungamaIE(InfoExtractor):
         thumbnail = data.get('img_src') or data.get('album_image')
 
         return {
-            'id': video_id,
+            'id': audio_id,
             'title': title,
             'thumbnail': thumbnail,
             'track': track,

From 6089ff40e7cc7710e399db1be87fea103a190ee6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Jan 2019 00:37:01 +0700
Subject: [PATCH 296/510] [youporn] Fix title and description extraction
 (closes #18748)

---
 youtube_dl/extractor/youporn.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index ea0bce784..d4eccb4b2 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -68,11 +68,9 @@ class YouPornIE(InfoExtractor):
         request.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(request, display_id)
 
-        title = self._search_regex(
-            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
-             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>(?P<title>[^<]+)<'],
-            webpage, 'title', group='title',
-            default=None) or self._og_search_title(
+        title = self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
+            webpage, 'title', default=None) or self._og_search_title(
             webpage, default=None) or self._html_search_meta(
             'title', webpage, fatal=True)
 
@@ -134,7 +132,11 @@ class YouPornIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
-        description = self._og_search_description(webpage, default=None)
+        description = self._html_search_regex(
+            r'(?s)<div[^>]+\bid=["\']description["\'][^>]*>(.+?)</div>',
+            webpage, 'description',
+            default=None) or self._og_search_description(
+            webpage, default=None)
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')

From 3c1089dba41f137dcc6c373daacdc19a24aef81c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 9 Jan 2019 14:23:26 +0100
Subject: [PATCH 297/510] [gaia] Add new extractor(#14605)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/gaia.py       | 98 ++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 youtube_dl/extractor/gaia.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ddeb70284..c7d04d366 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -411,6 +411,7 @@ from .funk import (
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
 from .fxnetworks import FXNetworksIE
+from .gaia import GaiaIE
 from .gameinformer import GameInformerIE
 from .gameone import (
     GameOneIE,
diff --git a/youtube_dl/extractor/gaia.py b/youtube_dl/extractor/gaia.py
new file mode 100644
index 000000000..f2eef3f4c
--- /dev/null
+++ b/youtube_dl/extractor/gaia.py
@@ -0,0 +1,98 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    str_or_none,
+    strip_or_none,
+    try_get,
+)
+
+
+class GaiaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gaia\.com/video/(?P<id>[^/?]+).*?\bfullplayer=(?P<type>feature|preview)'
+    _TESTS = [{
+        'url': 'https://www.gaia.com/video/connecting-universal-consciousness?fullplayer=feature',
+        'info_dict': {
+            'id': '89356',
+            'ext': 'mp4',
+            'title': 'Connecting with Universal Consciousness',
+            'description': 'md5:844e209ad31b7d31345f5ed689e3df6f',
+            'upload_date': '20151116',
+            'timestamp': 1447707266,
+            'duration': 936,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.gaia.com/video/connecting-universal-consciousness?fullplayer=preview',
+        'info_dict': {
+            'id': '89351',
+            'ext': 'mp4',
+            'title': 'Connecting with Universal Consciousness',
+            'description': 'md5:844e209ad31b7d31345f5ed689e3df6f',
+            'upload_date': '20151116',
+            'timestamp': 1447707266,
+            'duration': 53,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id, vtype = re.search(self._VALID_URL, url).groups()
+        node_id = self._download_json(
+            'https://brooklyn.gaia.com/pathinfo', display_id, query={
+                'path': 'video/' + display_id,
+            })['id']
+        node = self._download_json(
+            'https://brooklyn.gaia.com/node/%d' % node_id, node_id)
+        vdata = node[vtype]
+        media_id = compat_str(vdata['nid'])
+        title = node['title']
+
+        media = self._download_json(
+            'https://brooklyn.gaia.com/media/' + media_id, media_id)
+        formats = self._extract_m3u8_formats(
+            media['mediaUrls']['bcHLS'], media_id, 'mp4')
+        self._sort_formats(formats)
+
+        subtitles = {}
+        text_tracks = media.get('textTracks', {})
+        for key in ('captions', 'subtitles'):
+            for lang, sub_url in text_tracks.get(key, {}).items():
+                subtitles.setdefault(lang, []).append({
+                    'url': sub_url,
+                })
+
+        fivestar = node.get('fivestar', {})
+        fields = node.get('fields', {})
+
+        def get_field_value(key, value_key='value'):
+            return try_get(fields, lambda x: x[key][0][value_key])
+
+        return {
+            'id': media_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'description': strip_or_none(get_field_value('body') or get_field_value('teaser')),
+            'timestamp': int_or_none(node.get('created')),
+            'subtitles': subtitles,
+            'duration': int_or_none(vdata.get('duration')),
+            'like_count': int_or_none(try_get(fivestar, lambda x: x['up_count']['value'])),
+            'dislike_count': int_or_none(try_get(fivestar, lambda x: x['down_count']['value'])),
+            'comment_count': int_or_none(node.get('comment_count')),
+            'series': try_get(node, lambda x: x['series']['title'], compat_str),
+            'season_number': int_or_none(get_field_value('season')),
+            'season_id': str_or_none(get_field_value('series_nid', 'nid')),
+            'episode_number': int_or_none(get_field_value('episode')),
+        }

From 65615be36874fe98d5a972546e8024cda3879c86 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 9 Jan 2019 19:17:58 +0100
Subject: [PATCH 298/510] [globo] set GLBID cookie manually(closes #17346)

---
 youtube_dl/extractor/globo.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index c2140c362..fb8f7679b 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -72,7 +72,7 @@ class GloboIE(InfoExtractor):
             return
 
         try:
-            self._download_json(
+            glb_id = (self._download_json(
                 'https://login.globo.com/api/authentication', None, data=json.dumps({
                     'payload': {
                         'email': email,
@@ -81,7 +81,9 @@ class GloboIE(InfoExtractor):
                     },
                 }).encode(), headers={
                     'Content-Type': 'application/json; charset=utf-8',
-                })
+                }) or {}).get('glbId')
+            if glb_id:
+                self._set_cookie('.globo.com', 'GLBID', glb_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
                 resp = self._parse_json(e.cause.read(), None)

From 4ad159c7b0ce476171aad1eaa8a36f7b9b1cce65 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 9 Jan 2019 20:39:48 +0100
Subject: [PATCH 299/510] [playplustv] Add new extractor(closes #18789)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/playplustv.py | 109 +++++++++++++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100644 youtube_dl/extractor/playplustv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c7d04d366..7d7e4247a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -861,6 +861,7 @@ from .piksel import PikselIE
 from .pinkbike import PinkbikeIE
 from .pladform import PladformIE
 from .playfm import PlayFMIE
+from .playplustv import PlayPlusTVIE
 from .plays import PlaysTVIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
diff --git a/youtube_dl/extractor/playplustv.py b/youtube_dl/extractor/playplustv.py
new file mode 100644
index 000000000..419c2974b
--- /dev/null
+++ b/youtube_dl/extractor/playplustv.py
@@ -0,0 +1,109 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    int_or_none,
+    PUTRequest,
+)
+
+
+class PlayPlusTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?playplus\.tv/VOD/(?P<project_id>[0-9]+)/(?P<id>[0-9a-f]{32})'
+    _TEST = {
+        'url': 'https://www.playplus.tv/VOD/7572/db8d274a5163424e967f35a30ddafb8e',
+        'md5': 'd078cb89d7ab6b9df37ce23c647aef72',
+        'info_dict': {
+            'id': 'db8d274a5163424e967f35a30ddafb8e',
+            'ext': 'mp4',
+            'title': 'Capítulo 179 - Final',
+            'description': 'md5:01085d62d8033a1e34121d3c3cabc838',
+            'timestamp': 1529992740,
+            'upload_date': '20180626',
+        },
+        'skip': 'Requires account credential',
+    }
+    _NETRC_MACHINE = 'playplustv'
+    _GEO_COUNTRIES = ['BR']
+    _token = None
+    _profile_id = None
+
+    def _call_api(self, resource, video_id=None, query=None):
+        return self._download_json('https://api.playplus.tv/api/media/v2/get' + resource, video_id, headers={
+            'Authorization': 'Bearer ' + self._token,
+        }, query=query)
+
+    def _real_initialize(self):
+        email, password = self._get_login_info()
+        if email is None:
+            self.raise_login_required()
+
+        req = PUTRequest(
+            'https://api.playplus.tv/api/web/login', json.dumps({
+                'email': email,
+                'password': password,
+            }).encode(), {
+                'Content-Type': 'application/json; charset=utf-8',
+            })
+
+        try:
+            self._token = self._download_json(req, None)['token']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                raise ExtractorError(self._parse_json(
+                    e.cause.read(), None)['errorMessage'], expected=True)
+            raise
+
+        self._profile = self._call_api('Profiles')['list'][0]['_id']
+
+    def _real_extract(self, url):
+        project_id, media_id = re.match(self._VALID_URL, url).groups()
+        media = self._call_api(
+            'Media', media_id, {
+                'profileId': self._profile,
+                'projectId': project_id,
+                'mediaId': media_id,
+            })['obj']
+        title = media['title']
+
+        formats = []
+        for f in media.get('files', []):
+            f_url = f.get('url')
+            if not f_url:
+                continue
+            file_info = f.get('fileInfo') or {}
+            formats.append({
+                'url': f_url,
+                'width': int_or_none(file_info.get('width')),
+                'height': int_or_none(file_info.get('height')),
+            })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for thumb in media.get('thumbs', []):
+            thumb_url = thumb.get('url')
+            if not thumb_url:
+                continue
+            thumbnails.append({
+                'url': thumb_url,
+                'width': int_or_none(thumb.get('width')),
+                'height': int_or_none(thumb.get('height')),
+            })
+
+        return {
+            'id': media_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': clean_html(media.get('description')) or media.get('shortDescription'),
+            'timestamp': int_or_none(media.get('publishDate'), 1000),
+            'view_count': int_or_none(media.get('numberOfViews')),
+            'comment_count': int_or_none(media.get('numberOfComments')),
+            'tags': media.get('tags'),
+        }

From 96c186e1fd8af4943eeaa8c858f9cb0f15e8b265 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 10 Jan 2019 09:05:00 +0100
Subject: [PATCH 300/510] [fox] add support National Geographic(closes
 #17985)(closes #15333)(closes #14698)

---
 youtube_dl/extractor/extractors.py         |   6 +-
 youtube_dl/extractor/fox.py                |  93 +++++++-------
 youtube_dl/extractor/nationalgeographic.py | 135 ---------------------
 3 files changed, 48 insertions(+), 186 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7d7e4247a..a01d99b85 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -687,11 +687,7 @@ from .myvi import (
     MyviEmbedIE,
 )
 from .myvidster import MyVidsterIE
-from .nationalgeographic import (
-    NationalGeographicVideoIE,
-    NationalGeographicIE,
-    NationalGeographicEpisodeGuideIE,
-)
+from .nationalgeographic import NationalGeographicVideoIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 11d6c9c32..b1c91f095 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -1,11 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+# import json
+# import uuid
+
 from .adobepass import AdobePassIE
-from .uplynk import UplynkPreplayIE
-from ..compat import compat_str
 from ..utils import (
-    HEADRequest,
     int_or_none,
     parse_age_limit,
     parse_duration,
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class FOXIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[\da-fA-F]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:fox\.com|nationalgeographic\.com/tv)/watch/(?P<id>[\da-fA-F]+)'
     _TESTS = [{
         # clip
         'url': 'https://www.fox.com/watch/4b765a60490325103ea69888fb2bd4e8/',
@@ -43,41 +43,47 @@ class FOXIE(AdobePassIE):
         # episode, geo-restricted, tv provided required
         'url': 'https://www.fox.com/watch/30056b295fb57f7452aeeb4920bc3024/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.nationalgeographic.com/tv/watch/f690e05ebbe23ab79747becd0cc223d1/',
+        'only_matching': True,
     }]
+    # _access_token = None
+
+    # def _call_api(self, path, video_id, data=None):
+    #     headers = {
+    #         'X-Api-Key': '238bb0a0c2aba67922c48709ce0c06fd',
+    #     }
+    #     if self._access_token:
+    #         headers['Authorization'] = 'Bearer ' + self._access_token
+    #     return self._download_json(
+    #         'https://api2.fox.com/v2.0/' + path, video_id, data=data, headers=headers)
+
+    # def _real_initialize(self):
+    #     self._access_token = self._call_api(
+    #         'login', None, json.dumps({
+    #             'deviceId': compat_str(uuid.uuid4()),
+    #         }).encode())['accessToken']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'https://api.fox.com/fbc-content/v1_4/video/%s' % video_id,
+            'https://api.fox.com/fbc-content/v1_5/video/%s' % video_id,
             video_id, headers={
                 'apikey': 'abdcbed02c124d393b39e818a4312055',
                 'Content-Type': 'application/json',
                 'Referer': url,
             })
+        # video = self._call_api('vodplayer/' + video_id, video_id)
 
         title = video['name']
         release_url = video['videoRelease']['url']
-
-        description = video.get('description')
-        duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
-            video.get('duration')) or parse_duration(video.get('duration'))
-        timestamp = unified_timestamp(video.get('datePublished'))
-        rating = video.get('contentRating')
-        age_limit = parse_age_limit(rating)
+        # release_url = video['url']
 
         data = try_get(
             video, lambda x: x['trackingData']['properties'], dict) or {}
 
-        creator = data.get('brand') or data.get('network') or video.get('network')
-
-        series = video.get('seriesName') or data.get(
-            'seriesName') or data.get('show')
-        season_number = int_or_none(video.get('seasonNumber'))
-        episode = video.get('name')
-        episode_number = int_or_none(video.get('episodeNumber'))
-        release_year = int_or_none(video.get('releaseYear'))
-
+        rating = video.get('contentRating')
         if data.get('authRequired'):
             resource = self._get_mvpd_resource(
                 'fbc-fox', title, video.get('guid'), rating)
@@ -86,6 +92,18 @@ class FOXIE(AdobePassIE):
                     'auth': self._extract_mvpd_auth(
                         url, video_id, 'fbc-fox', resource)
                 })
+        m3u8_url = self._download_json(release_url, video_id)['playURL']
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
+            video.get('duration')) or parse_duration(video.get('duration'))
+        timestamp = unified_timestamp(video.get('datePublished'))
+        creator = data.get('brand') or data.get('network') or video.get('network')
+        series = video.get('seriesName') or data.get(
+            'seriesName') or data.get('show')
 
         subtitles = {}
         for doc_rel in video.get('documentReleases', []):
@@ -98,36 +116,19 @@ class FOXIE(AdobePassIE):
             }]
             break
 
-        info = {
+        return {
             'id': video_id,
             'title': title,
-            'description': description,
+            'formats': formats,
+            'description': video.get('description'),
             'duration': duration,
             'timestamp': timestamp,
-            'age_limit': age_limit,
+            'age_limit': parse_age_limit(rating),
             'creator': creator,
             'series': series,
-            'season_number': season_number,
-            'episode': episode,
-            'episode_number': episode_number,
-            'release_year': release_year,
+            'season_number': int_or_none(video.get('seasonNumber')),
+            'episode': video.get('name'),
+            'episode_number': int_or_none(video.get('episodeNumber')),
+            'release_year': int_or_none(video.get('releaseYear')),
             'subtitles': subtitles,
         }
-
-        urlh = self._request_webpage(HEADRequest(release_url), video_id)
-        video_url = compat_str(urlh.geturl())
-
-        if UplynkPreplayIE.suitable(video_url):
-            info.update({
-                '_type': 'url_transparent',
-                'url': video_url,
-                'ie_key': UplynkPreplayIE.ie_key(),
-            })
-        else:
-            m3u8_url = self._download_json(release_url, video_id)['playURL']
-            formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4',
-                entry_protocol='m3u8_native', m3u8_id='hls')
-            self._sort_formats(formats)
-            info['formats'] = formats
-        return info
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 4d2ee6408..165964ca0 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -1,15 +1,9 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from .adobepass import AdobePassIE
-from .theplatform import ThePlatformIE
 from ..utils import (
     smuggle_url,
     url_basename,
-    update_url_query,
-    get_element_by_class,
 )
 
 
@@ -64,132 +58,3 @@ class NationalGeographicVideoIE(InfoExtractor):
                 {'force_smil_url': True}),
             'id': guid,
         }
-
-
-class NationalGeographicIE(ThePlatformIE, AdobePassIE):
-    IE_NAME = 'natgeo'
-    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:(?:(?:wild/)?[^/]+/)?(?:videos|episodes)|u)/(?P<id>[^/?]+)'
-
-    _TESTS = [
-        {
-            'url': 'http://channel.nationalgeographic.com/u/kdi9Ld0PN2molUUIMSBGxoeDhD729KRjQcnxtetilWPMevo8ZwUBIDuPR0Q3D2LVaTsk0MPRkRWDB8ZhqWVeyoxfsZZm36yRp1j-zPfsHEyI_EgAeFY/',
-            'md5': '518c9aa655686cf81493af5cc21e2a04',
-            'info_dict': {
-                'id': 'vKInpacll2pC',
-                'ext': 'mp4',
-                'title': 'Uncovering a Universal Knowledge',
-                'description': 'md5:1a89148475bf931b3661fcd6ddb2ae3a',
-                'timestamp': 1458680907,
-                'upload_date': '20160322',
-                'uploader': 'NEWA-FNG-NGTV',
-            },
-            'add_ie': ['ThePlatform'],
-        },
-        {
-            'url': 'http://channel.nationalgeographic.com/u/kdvOstqYaBY-vSBPyYgAZRUL4sWUJ5XUUPEhc7ISyBHqoIO4_dzfY3K6EjHIC0hmFXoQ7Cpzm6RkET7S3oMlm6CFnrQwSUwo/',
-            'md5': 'c4912f656b4cbe58f3e000c489360989',
-            'info_dict': {
-                'id': 'Pok5lWCkiEFA',
-                'ext': 'mp4',
-                'title': 'The Stunning Red Bird of Paradise',
-                'description': 'md5:7bc8cd1da29686be4d17ad1230f0140c',
-                'timestamp': 1459362152,
-                'upload_date': '20160330',
-                'uploader': 'NEWA-FNG-NGTV',
-            },
-            'add_ie': ['ThePlatform'],
-        },
-        {
-            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/episodes/the-power-of-miracles/',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://channel.nationalgeographic.com/videos/treasures-rediscovered/',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/videos/uncovering-a-universal-knowledge/',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://channel.nationalgeographic.com/wild/destination-wild/videos/the-stunning-red-bird-of-paradise/',
-            'only_matching': True,
-        }
-    ]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        release_url = self._search_regex(
-            r'video_auth_playlist_url\s*=\s*"([^"]+)"',
-            webpage, 'release url')
-        theplatform_path = self._search_regex(r'https?://link\.theplatform\.com/s/([^?]+)', release_url, 'theplatform path')
-        video_id = theplatform_path.split('/')[-1]
-        query = {
-            'mbr': 'true',
-        }
-        is_auth = self._search_regex(r'video_is_auth\s*=\s*"([^"]+)"', webpage, 'is auth', fatal=False)
-        if is_auth == 'auth':
-            auth_resource_id = self._search_regex(
-                r"video_auth_resourceId\s*=\s*'([^']+)'",
-                webpage, 'auth resource id')
-            query['auth'] = self._extract_mvpd_auth(url, video_id, 'natgeo', auth_resource_id)
-
-        formats = []
-        subtitles = {}
-        for key, value in (('switch', 'http'), ('manifest', 'm3u')):
-            tp_query = query.copy()
-            tp_query.update({
-                key: value,
-            })
-            tp_formats, tp_subtitles = self._extract_theplatform_smil(
-                update_url_query(release_url, tp_query), video_id, 'Downloading %s SMIL data' % value)
-            formats.extend(tp_formats)
-            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
-        self._sort_formats(formats)
-
-        info = self._extract_theplatform_metadata(theplatform_path, display_id)
-        info.update({
-            'id': video_id,
-            'formats': formats,
-            'subtitles': subtitles,
-            'display_id': display_id,
-        })
-        return info
-
-
-class NationalGeographicEpisodeGuideIE(InfoExtractor):
-    IE_NAME = 'natgeo:episodeguide'
-    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?(?P<id>[^/]+)/episode-guide'
-    _TESTS = [
-        {
-            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/episode-guide/',
-            'info_dict': {
-                'id': 'the-story-of-god-with-morgan-freeman-season-1',
-                'title': 'The Story of God with Morgan Freeman - Season 1',
-            },
-            'playlist_mincount': 6,
-        },
-        {
-            'url': 'http://channel.nationalgeographic.com/underworld-inc/episode-guide/?s=2',
-            'info_dict': {
-                'id': 'underworld-inc-season-2',
-                'title': 'Underworld, Inc. - Season 2',
-            },
-            'playlist_mincount': 7,
-        },
-    ]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        show = get_element_by_class('show', webpage)
-        selected_season = self._search_regex(
-            r'<div[^>]+class="select-seasons[^"]*".*?<a[^>]*>(.*?)</a>',
-            webpage, 'selected season')
-        entries = [
-            self.url_result(self._proto_relative_url(entry_url), 'NationalGeographic')
-            for entry_url in re.findall('(?s)<div[^>]+class="col-inner"[^>]*?>.*?<a[^>]+href="([^"]+)"', webpage)]
-        return self.playlist_result(
-            entries, '%s-%s' % (display_id, selected_season.lower().replace(' ', '-')),
-            '%s - %s' % (show, selected_season))

From 7c072f00d66f3b3f44d301059443480a5c83d3c0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 10 Jan 2019 10:50:18 +0100
Subject: [PATCH 301/510] [jwplatform] use JW Platform Delivery API V2 and add
 support for more urls

---
 youtube_dl/extractor/jwplatform.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 63d0dc998..d19a6a774 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 
 
 class JWPlatformIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
-    _TEST = {
+    _VALID_URL = r'(?:https?://(?:content\.jwplatform|cdn\.jwplayer)\.com/(?:(?:feed|player|thumb|preview|video|manifest)s|jw6|v2/media)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
+    _TESTS = [{
         'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
         'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
         'info_dict': {
@@ -19,7 +19,10 @@ class JWPlatformIE(InfoExtractor):
             'upload_date': '20081127',
             'timestamp': 1227796140,
         }
-    }
+    }, {
+        'url': 'https://cdn.jwplayer.com/players/nPripu9l-ALJ3XQCI.js',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def _extract_url(webpage):
@@ -34,5 +37,5 @@ class JWPlatformIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
+        json_data = self._download_json('https://cdn.jwplayer.com/v2/media/' + video_id, video_id)
         return self._parse_jwplayer_data(json_data, video_id)

From 432aba1c5e68cf8efccb868107785a15617d5109 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 10 Jan 2019 10:54:46 +0100
Subject: [PATCH 302/510] [outsidetv] Add new extractor(closes #18774)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/outsidetv.py  | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/outsidetv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a01d99b85..de38c6641 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -829,6 +829,7 @@ from .orf import (
     ORFOE1IE,
     ORFIPTVIE,
 )
+from .outsidetv import OutsideTVIE
 from .packtpub import (
     PacktPubIE,
     PacktPubCourseIE,
diff --git a/youtube_dl/extractor/outsidetv.py b/youtube_dl/extractor/outsidetv.py
new file mode 100644
index 000000000..c5333b08c
--- /dev/null
+++ b/youtube_dl/extractor/outsidetv.py
@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class OutsideTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?outsidetv\.com/(?:[^/]+/)*?play/[a-zA-Z0-9]{8}/\d+/\d+/(?P<id>[a-zA-Z0-9]{8})'
+    _TESTS = [{
+        'url': 'http://www.outsidetv.com/category/snow/play/ZjQYboH6/1/10/Hdg0jukV/4',
+        'md5': '192d968fedc10b2f70ec31865ffba0da',
+        'info_dict': {
+            'id': 'Hdg0jukV',
+            'ext': 'mp4',
+            'title': 'Home - Jackson Ep 1 | Arbor Snowboards',
+            'description': 'md5:41a12e94f3db3ca253b04bb1e8d8f4cd',
+            'upload_date': '20181225',
+            'timestamp': 1545742800,
+        }
+    }, {
+        'url': 'http://www.outsidetv.com/home/play/ZjQYboH6/1/10/Hdg0jukV/4',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        jw_media_id = self._match_id(url)
+        return self.url_result(
+            'jwplatform:' + jw_media_id, 'JWPlatform', jw_media_id)

From c3e543893bfba7faa7c13e53fbe6b60f936b81f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Jan 2019 22:46:53 +0700
Subject: [PATCH 303/510] [youtube] Extract live HLS URL from player response
 (closes #18799)

---
 youtube_dl/extractor/youtube.py | 55 +++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 539ff6ff2..29773877e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1931,31 +1931,38 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'http_chunk_size': 10485760,
                     }
                 formats.append(dct)
-        elif video_info.get('hlsvp'):
-            manifest_url = video_info['hlsvp'][0]
-            formats = []
-            m3u8_formats = self._extract_m3u8_formats(
-                manifest_url, video_id, 'mp4', fatal=False)
-            for a_format in m3u8_formats:
-                itag = self._search_regex(
-                    r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
-                if itag:
-                    a_format['format_id'] = itag
-                    if itag in self._formats:
-                        dct = self._formats[itag].copy()
-                        dct.update(a_format)
-                        a_format = dct
-                a_format['player_url'] = player_url
-                # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
-                a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
-                formats.append(a_format)
         else:
-            error_message = clean_html(video_info.get('reason', [None])[0])
-            if not error_message:
-                error_message = extract_unavailable_message()
-            if error_message:
-                raise ExtractorError(error_message, expected=True)
-            raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
+            manifest_url = (
+                url_or_none(try_get(
+                    player_response,
+                    lambda x: x['streamingData']['hlsManifestUrl'],
+                    compat_str)) or
+                url_or_none(try_get(
+                    video_info, lambda x: x['hlsvp'][0], compat_str)))
+            if manifest_url:
+                formats = []
+                m3u8_formats = self._extract_m3u8_formats(
+                    manifest_url, video_id, 'mp4', fatal=False)
+                for a_format in m3u8_formats:
+                    itag = self._search_regex(
+                        r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
+                    if itag:
+                        a_format['format_id'] = itag
+                        if itag in self._formats:
+                            dct = self._formats[itag].copy()
+                            dct.update(a_format)
+                            a_format = dct
+                    a_format['player_url'] = player_url
+                    # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
+                    a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
+                    formats.append(a_format)
+            else:
+                error_message = clean_html(video_info.get('reason', [None])[0])
+                if not error_message:
+                    error_message = extract_unavailable_message()
+                if error_message:
+                    raise ExtractorError(error_message, expected=True)
+                raise ExtractorError('no conn, hlsvp, hlsManifestUrl or url_encoded_fmt_stream_map information found in video info')
 
         # uploader
         video_uploader = try_get(

From a4491dd55c5feaf02ac8969caafd637265cbeede Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Jan 2019 23:23:19 +0700
Subject: [PATCH 304/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 41d190a72..c7d090c6c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,29 @@
+version <unreleased>
+
+Core
+* [extractor/common] Use episode name as title in _json_ld
++ [extractor/common] Add support for movies in _json_ld
+* [postprocessor/ffmpeg] Embed subtitles with non-standard language codes
+  (#18765)
++ [utils] Add language codes replaced in 1989 revision of ISO 639
+  to ISO639Utils (#18765)
+
+Extractors
+* [youtube] Extract live HLS URL from player response (#18799)
++ [outsidetv] Add support for outsidetv.com (#18774)
+* [jwplatform] Use JW Platform Delivery API V2 and add support for more URLs
++ [fox] Add support National Geographic (#17985, #15333, #14698)
++ [playplustv] Add support for playplus.tv (#18789)
+* [globo] Set GLBID cookie manually (#17346)
++ [gaia] Add support for gaia.com (#14605)
+* [youporn] Fix title and description extraction (#18748)
++ [hungama] Add support for hungama.com (#17402, #18771)
+* [dtube] Fix extraction (#18741)
+* [tvnow] Fix and rework extractors and prepare for a switch to the new API
+  (#17245, #18499)
+* [carambatv:page] Fix extraction (#18739)
+
+
 version 2019.01.02
 
 Extractors

From b64f6e690ff84052751c593a8826a40a63cb1d84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Jan 2019 23:26:54 +0700
Subject: [PATCH 305/510] release 2019.01.10

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 11 ++++++++---
 youtube_dl/version.py     |  2 +-
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3986bbc8e..d6abdbcb7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.02
+[debug] youtube-dl version 2019.01.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c7d090c6c..3d60754c5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.10
 
 Core
 * [extractor/common] Use episode name as title in _json_ld
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e92064432..c01409419 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -320,6 +320,7 @@
  - **Fusion**
  - **Fux**
  - **FXNetworks**
+ - **Gaia**
  - **GameInformer**
  - **GameOne**
  - **gameone:playlist**
@@ -370,6 +371,8 @@
  - **HRTiPlaylist**
  - **Huajiao**: 花椒直播
  - **HuffPost**: Huffington Post
+ - **Hungama**
+ - **HungamaSong**
  - **Hypem**
  - **Iconosquare**
  - **ign.com**
@@ -540,8 +543,6 @@
  - **MyviEmbed**
  - **MyVisionTV**
  - **n-tv.de**
- - **natgeo**
- - **natgeo:episodeguide**
  - **natgeo:video**
  - **Naver**
  - **NBA**
@@ -642,6 +643,7 @@
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
  - **OsnatelTV**
+ - **OutsideTV**
  - **PacktPub**
  - **PacktPubCourse**
  - **PandaTV**: 熊猫TV
@@ -666,6 +668,7 @@
  - **Pinkbike**
  - **Pladform**
  - **play.fm**
+ - **PlayPlusTV**
  - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
@@ -934,7 +937,9 @@
  - **TVNet**
  - **TVNoe**
  - **TVNow**
- - **TVNowList**
+ - **TVNowAnnual**
+ - **TVNowNew**
+ - **TVNowSeason**
  - **TVNowShow**
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 07a444706..5ba61f489 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.02'
+__version__ = '2019.01.10'

From c469e8808c723d6ba090c0b33c292622a3cb555e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 10 Jan 2019 18:48:52 +0100
Subject: [PATCH 306/510] [playplustv] add support for playplus.com(#18789)

---
 youtube_dl/extractor/playplustv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playplustv.py b/youtube_dl/extractor/playplustv.py
index 419c2974b..1e30ab23a 100644
--- a/youtube_dl/extractor/playplustv.py
+++ b/youtube_dl/extractor/playplustv.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class PlayPlusTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?playplus\.tv/VOD/(?P<project_id>[0-9]+)/(?P<id>[0-9a-f]{32})'
+    _VALID_URL = r'https?://(?:www\.)?playplus\.(?:com|tv)/VOD/(?P<project_id>[0-9]+)/(?P<id>[0-9a-f]{32})'
     _TEST = {
         'url': 'https://www.playplus.tv/VOD/7572/db8d274a5163424e967f35a30ddafb8e',
         'md5': 'd078cb89d7ab6b9df37ce23c647aef72',

From a64646e417615173c9436887116402c9c4c9f6ee Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 11 Jan 2019 15:09:44 +0100
Subject: [PATCH 307/510] [postprocessor/ffmpeg] sanitize ffmpeg version for
 Ubuntu and Arch Linux systems(closes #18813)

---
 youtube_dl/postprocessor/ffmpeg.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index efcd39d57..173bdbe68 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -79,6 +79,19 @@ class FFmpegPostProcessor(PostProcessor):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         prefer_ffmpeg = True
 
+        def get_ffmpeg_version(path):
+            ver = get_exe_version(path, args=['-version'])
+            if ver:
+                regexs = [
+                    r'([0-9.]+)-0ubuntu0\.[0-9.]+$', # Ubuntu
+                    r'n([0-9.]+)$', # Arch Linux
+                ]
+                for regex in regexs:
+                    mobj = re.match(regex, ver)
+                    if mobj:
+                        ver = mobj.group(1)
+            return ver
+
         self.basename = None
         self.probe_basename = None
 
@@ -110,11 +123,10 @@ class FFmpegPostProcessor(PostProcessor):
                 self._paths = dict(
                     (p, os.path.join(location, p)) for p in programs)
                 self._versions = dict(
-                    (p, get_exe_version(self._paths[p], args=['-version']))
-                    for p in programs)
+                    (p, get_ffmpeg_version(self._paths[p])) for p in programs)
         if self._versions is None:
             self._versions = dict(
-                (p, get_exe_version(p, args=['-version'])) for p in programs)
+                (p, get_ffmpeg_version(p)) for p in programs)
             self._paths = dict((p, p) for p in programs)
 
         if prefer_ffmpeg is False:

From 5caa531a1a7e070e3bef8cc60c3d00c3f7ab1e6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Jan 2019 23:47:23 +0700
Subject: [PATCH 308/510] [postprocessor/ffmpeg] PEP 8

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 173bdbe68..a9fafa363 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -83,8 +83,8 @@ class FFmpegPostProcessor(PostProcessor):
             ver = get_exe_version(path, args=['-version'])
             if ver:
                 regexs = [
-                    r'([0-9.]+)-0ubuntu0\.[0-9.]+$', # Ubuntu
-                    r'n([0-9.]+)$', # Arch Linux
+                    r'([0-9.]+)-0ubuntu0\.[0-9.]+$',  # Ubuntu
+                    r'n([0-9.]+)$',  # Arch Linux
                 ]
                 for regex in regexs:
                     mobj = re.match(regex, ver)

From cbdc688c41876714f1d7e2f82ba9d11f6695448a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Jan 2019 00:30:06 +0700
Subject: [PATCH 309/510] [postprocessor/ffmpeg] Relax ubuntu ffmpeg version
 regex

---
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a9fafa363..39a905380 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -83,8 +83,9 @@ class FFmpegPostProcessor(PostProcessor):
             ver = get_exe_version(path, args=['-version'])
             if ver:
                 regexs = [
-                    r'([0-9.]+)-0ubuntu0\.[0-9.]+$',  # Ubuntu
+                    r'(?:\d+:)?([0-9.]+)-[0-9]+ubuntu[0-9.]+$',  # Ubuntu, see [1]
                     r'n([0-9.]+)$',  # Arch Linux
+                    # 1. http://www.ducea.com/2006/06/17/ubuntu-package-version-naming-explanation/
                 ]
                 for regex in regexs:
                     mobj = re.match(regex, ver)

From 60a899bb7ed583b4c40289925c8127e746c683ea Mon Sep 17 00:00:00 2001
From: Atlas Sullivan <AtlasJan@gmx.com>
Date: Fri, 11 Jan 2019 19:15:48 +0000
Subject: [PATCH 310/510] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 901595444..70bcfaccf 100644
--- a/README.md
+++ b/README.md
@@ -496,7 +496,7 @@ The `-o` option allows users to indicate a template for the output file names.
 
 **tl;dr:** [navigate me to examples](#output-template-examples).
 
-The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a formatting operations. Allowed names along with sequence type are:
+The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. Allowed names along with sequence type are:
 
  - `id` (string): Video identifier
  - `title` (string): Video title

From ed8db0a25c93f5611dfae81ee8f15cbc177bd0ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Jan 2019 04:56:17 +0700
Subject: [PATCH 311/510] [wistia] Extend _VALID_URL (closes #18823)

---
 youtube_dl/extractor/wistia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 01a51275e..fa142b974 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class WistiaIE(InfoExtractor):
-    _VALID_URL = r'(?:wistia:|https?://(?:fast\.)?wistia\.(?:net|com)/embed/iframe/)(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'(?:wistia:|https?://(?:fast\.)?wistia\.(?:net|com)/embed/(?:iframe|medias)/)(?P<id>[a-z0-9]+)'
     _API_URL = 'http://fast.wistia.com/embed/medias/%s.json'
     _IFRAME_URL = 'http://fast.wistia.net/embed/iframe/%s'
 
@@ -38,6 +38,9 @@ class WistiaIE(InfoExtractor):
     }, {
         'url': 'http://fast.wistia.com/embed/iframe/sh7fpupwlt',
         'only_matching': True,
+    }, {
+        'url': 'http://fast.wistia.net/embed/medias/sh7fpupwlt.json',
+        'only_matching': True,
     }]
 
     @staticmethod

From d65f6e734b5c00acbb396f4569907f7957e1cbef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Jan 2019 03:57:31 +0700
Subject: [PATCH 312/510] [bitchute] Check formats (#18833)

---
 youtube_dl/extractor/bitchute.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index aa034355a..4f39424f5 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -55,6 +55,7 @@ class BitChuteIE(InfoExtractor):
         formats = [
             {'url': format_url}
             for format_url in orderedSet(format_urls)]
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         description = self._html_search_regex(

From f1ab3b7de7667c0382bf6eb6364ccab4260c0654 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 13 Jan 2019 10:01:26 +0100
Subject: [PATCH 313/510] [downloader/hls] fix uplynk ad skipping(closes
 #18824)

---
 youtube_dl/downloader/hls.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index fd304527e..4def8e2d5 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -75,10 +75,14 @@ class HlsFD(FragmentFD):
                 fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
-        def is_ad_fragment(s):
+        def is_ad_fragment_start(s):
             return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s or
                     s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))
 
+        def is_ad_fragment_end(s):
+            return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s or
+                    s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment'))
+
         media_frags = 0
         ad_frags = 0
         ad_frag_next = False
@@ -87,12 +91,13 @@ class HlsFD(FragmentFD):
             if not line:
                 continue
             if line.startswith('#'):
-                if is_ad_fragment(line):
-                    ad_frags += 1
+                if is_ad_fragment_start(line):
                     ad_frag_next = True
+                elif is_ad_fragment_end(line):
+                    ad_frag_next = False
                 continue
             if ad_frag_next:
-                ad_frag_next = False
+                ad_frags += 1
                 continue
             media_frags += 1
 
@@ -123,7 +128,6 @@ class HlsFD(FragmentFD):
             if line:
                 if not line.startswith('#'):
                     if ad_frag_next:
-                        ad_frag_next = False
                         continue
                     frag_index += 1
                     if frag_index <= ctx['fragment_index']:
@@ -196,8 +200,10 @@ class HlsFD(FragmentFD):
                         'start': sub_range_start,
                         'end': sub_range_start + int(splitted_byte_range[0]),
                     }
-                elif is_ad_fragment(line):
+                elif is_ad_fragment_start(line):
                     ad_frag_next = True
+                elif is_ad_fragment_end(line):
+                    ad_frag_next = False
 
         self._finish_frag_download(ctx)
 

From 3b983ee471305946709dcb83fa3799fc26a7db03 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 13 Jan 2019 15:46:54 +0100
Subject: [PATCH 314/510] [curiositystream] add support for non app urls

---
 youtube_dl/extractor/curiositystream.py | 56 +++++++++++++------------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index 35b1e7a34..e4a7fca6c 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -46,8 +46,24 @@ class CuriosityStreamBaseIE(InfoExtractor):
         self._handle_errors(result)
         self._auth_token = result['message']['auth_token']
 
-    def _extract_media_info(self, media):
-        video_id = compat_str(media['id'])
+
+class CuriosityStreamIE(CuriosityStreamBaseIE):
+    IE_NAME = 'curiositystream'
+    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://app.curiositystream.com/video/2',
+        'md5': '262bb2f257ff301115f1973540de8983',
+        'info_dict': {
+            'id': '2',
+            'ext': 'mp4',
+            'title': 'How Did You Develop The Internet?',
+            'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        media = self._call_api('media/' + video_id, video_id)
         title = media['title']
 
         formats = []
@@ -114,38 +130,21 @@ class CuriosityStreamBaseIE(InfoExtractor):
         }
 
 
-class CuriosityStreamIE(CuriosityStreamBaseIE):
-    IE_NAME = 'curiositystream'
-    _VALID_URL = r'https?://app\.curiositystream\.com/video/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://app.curiositystream.com/video/2',
-        'md5': '262bb2f257ff301115f1973540de8983',
-        'info_dict': {
-            'id': '2',
-            'ext': 'mp4',
-            'title': 'How Did You Develop The Internet?',
-            'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        media = self._call_api('media/' + video_id, video_id)
-        return self._extract_media_info(media)
-
-
 class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
     IE_NAME = 'curiositystream:collection'
-    _VALID_URL = r'https?://app\.curiositystream\.com/collection/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:collection|series)/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://app.curiositystream.com/collection/2',
         'info_dict': {
             'id': '2',
             'title': 'Curious Minds: The Internet',
             'description': 'How is the internet shaping our lives in the 21st Century?',
         },
-        'playlist_mincount': 12,
-    }
+        'playlist_mincount': 17,
+    }, {
+        'url': 'https://curiositystream.com/series/2',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         collection_id = self._match_id(url)
@@ -153,7 +152,10 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
             'collections/' + collection_id, collection_id)
         entries = []
         for media in collection.get('media', []):
-            entries.append(self._extract_media_info(media))
+            media_id = compat_str(media.get('id'))
+            entries.append(self.url_result(
+                'https://curiositystream.com/video/' + media_id,
+                CuriosityStreamIE.ie_key(), media_id))
         return self.playlist_result(
             entries, collection_id,
             collection.get('title'), collection.get('description'))

From 10026329c2534635876921da229f382098b8f8e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Jan 2019 23:23:51 +0700
Subject: [PATCH 315/510] [skylinewebcams] Fix extraction (closes #18853)

---
 youtube_dl/extractor/skylinewebcams.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/skylinewebcams.py b/youtube_dl/extractor/skylinewebcams.py
index 5b4aaac6f..b7f8ac736 100644
--- a/youtube_dl/extractor/skylinewebcams.py
+++ b/youtube_dl/extractor/skylinewebcams.py
@@ -26,7 +26,7 @@ class SkylineWebcamsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         stream_url = self._search_regex(
-            r'url\s*:\s*(["\'])(?P<url>(?:https?:)?//.+?\.m3u8.*?)\1', webpage,
+            r'(?:url|source)\s*:\s*(["\'])(?P<url>(?:https?:)?//.+?\.m3u8.*?)\1', webpage,
             'stream url', group='url')
 
         title = self._og_search_title(webpage)

From 929ba3997b026fd9fafe15d447732d8ccdf367bb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 15 Jan 2019 10:23:59 +0100
Subject: [PATCH 316/510] [funimation] fix extraction(closes #14089)

---
 youtube_dl/extractor/funimation.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 07d01caec..8bbedca26 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -1,6 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
+import string
+
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
 from ..utils import (
@@ -87,7 +90,7 @@ class FunimationIE(InfoExtractor):
 
         video_id = title_data.get('id') or self._search_regex([
             r"KANE_customdimensions.videoID\s*=\s*'(\d+)';",
-            r'<iframe[^>]+src="/player/(\d+)"',
+            r'<iframe[^>]+src="/player/(\d+)',
         ], webpage, 'video_id', default=None)
         if not video_id:
             player_url = self._html_search_meta([
@@ -108,8 +111,10 @@ class FunimationIE(InfoExtractor):
             if self._TOKEN:
                 headers['Authorization'] = 'Token %s' % self._TOKEN
             sources = self._download_json(
-                'https://prod-api-funimationnow.dadcdigital.com/api/source/catalog/video/%s/signed/' % video_id,
-                video_id, headers=headers)['items']
+                'https://www.funimation.com/api/showexperience/%s/' % video_id,
+                video_id, headers=headers, query={
+                    'pinst_id': ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(8)]),
+                })['items']
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 error = self._parse_json(e.cause.read(), video_id)['errors'][0]

From 561b456e2d10e60b74a1bee6154bb7fdb0bfaf60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Jan 2019 01:12:58 +0700
Subject: [PATCH 317/510] [youtube] Extract DASH formats from player response
 (closes #18804)

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 29773877e..cca149107 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1545,6 +1545,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if dash_mpd and dash_mpd[0] not in dash_mpds:
                 dash_mpds.append(dash_mpd[0])
 
+        def add_dash_mpd_pr(pl_response):
+            dash_mpd = url_or_none(try_get(
+                pl_response, lambda x: x['streamingData']['dashManifestUrl'],
+                compat_str))
+            if dash_mpd and dash_mpd not in dash_mpds:
+                dash_mpds.append(dash_mpd)
+
         is_live = None
         view_count = None
 
@@ -1602,6 +1609,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         if isinstance(pl_response, dict):
                             player_response = pl_response
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
+                add_dash_mpd_pr(player_response)
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
                 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
@@ -1633,6 +1641,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         pl_response = get_video_info.get('player_response', [None])[0]
                         if isinstance(pl_response, dict):
                             player_response = pl_response
+                            add_dash_mpd_pr(player_response)
                     add_dash_mpd(get_video_info)
                     if view_count is None:
                         view_count = extract_view_count(get_video_info)

From 2f483bc1c389709623117079439708783122b5ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Jan 2019 01:28:50 +0700
Subject: [PATCH 318/510] [youtube] Skip unsupported adaptive stream type
 (#18804)

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cca149107..5e93b5329 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1827,6 +1827,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 url_data = compat_parse_qs(url_data_str)
                 if 'itag' not in url_data or 'url' not in url_data:
                     continue
+                stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0]))
+                # Unsupported FORMAT_STREAM_TYPE_OTF
+                if stream_type == 3:
+                    continue
                 format_id = url_data['itag'][0]
                 url = url_data['url'][0]
 

From a16c7c033a161f310b69d444edc9dbf67cfc49ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Jan 2019 02:17:49 +0700
Subject: [PATCH 319/510] [test/helper] Add support for maxcount and count
 collection len test checkers

---
 test/helper.py | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index aa9a1c9b2..e62aab11e 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -153,15 +153,27 @@ def expect_value(self, got, expected, field):
                 isinstance(got, compat_str),
                 'Expected field %s to be a unicode object, but got value %r of type %r' % (field, got, type(got)))
             got = 'md5:' + md5(got)
-        elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+        elif isinstance(expected, compat_str) and re.match(r'^(?:min|max)?count:\d+', expected):
             self.assertTrue(
                 isinstance(got, (list, dict)),
                 'Expected field %s to be a list or a dict, but it is of type %s' % (
                     field, type(got).__name__))
-            expected_num = int(expected.partition(':')[2])
-            assertGreaterEqual(
+            op, _, expected_num = expected.partition(':')
+            expected_num = int(expected_num)
+            if op == 'mincount':
+                assert_func = assertGreaterEqual
+                msg_tmpl = 'Expected %d items in field %s, but only got %d'
+            elif op == 'maxcount':
+                assert_func = assertLessEqual
+                msg_tmpl = 'Expected maximum %d items in field %s, but got %d'
+            elif op == 'count':
+                assert_func = assertEqual
+                msg_tmpl = 'Expected exactly %d items in field %s, but got %d'
+            else:
+                assert False
+            assert_func(
                 self, len(got), expected_num,
-                'Expected %d items in field %s, but only got %d' % (expected_num, field, len(got)))
+                msg_tmpl % (expected_num, field, len(got)))
             return
         self.assertEqual(
             expected, got,
@@ -237,6 +249,20 @@ def assertGreaterEqual(self, got, expected, msg=None):
         self.assertTrue(got >= expected, msg)
 
 
+def assertLessEqual(self, got, expected, msg=None):
+    if not (got <= expected):
+        if msg is None:
+            msg = '%r not less than or equal to %r' % (got, expected)
+        self.assertTrue(got <= expected, msg)
+
+
+def assertEqual(self, got, expected, msg=None):
+    if not (got == expected):
+        if msg is None:
+            msg = '%r not equal to %r' % (got, expected)
+        self.assertTrue(got == expected, msg)
+
+
 def expect_warnings(ydl, warnings_re):
     real_warning = ydl.report_warning
 

From 4fe54c128a11d394874505af75aaa5a2276aa3ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Jan 2019 02:18:27 +0700
Subject: [PATCH 320/510] [youtube] Update tests and add a tests for #18804

---
 youtube_dl/extractor/youtube.py | 57 +++++++++++++++++----------------
 1 file changed, 29 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5e93b5329..730935657 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -498,7 +498,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
                 'upload_date': '20121002',
-                'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
@@ -527,7 +526,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
-                'license': 'Standard YouTube License',
                 'creator': 'Icona Pop',
                 'track': 'I Love It (feat. Charli XCX)',
                 'artist': 'Icona Pop',
@@ -540,14 +538,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': '07FYdnEawAQ',
                 'ext': 'mp4',
                 'upload_date': '20130703',
-                'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
+                'title': 'Justin Timberlake - Tunnel Vision (Official Music Video) (Explicit)',
                 'alt_title': 'Tunnel Vision',
-                'description': 'md5:64249768eec3bc4276236606ea996373',
+                'description': 'md5:07dab3356cde4199048e4c7cd93471e1',
                 'duration': 419,
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
-                'license': 'Standard YouTube License',
                 'creator': 'Justin Timberlake',
                 'track': 'Tunnel Vision',
                 'artist': 'Justin Timberlake',
@@ -566,7 +563,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'SET India',
                 'uploader_id': 'setindia',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
-                'license': 'Standard YouTube License',
                 'age_limit': 18,
             }
         },
@@ -581,7 +577,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'phihag',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'upload_date': '20121002',
-                'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
@@ -605,7 +600,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
                 'description': '',
                 'uploader': '8KVIDEO',
-                'license': 'Standard YouTube License',
                 'title': 'UHDTV TEST 8K VIDEO.mp4'
             },
             'params': {
@@ -620,13 +614,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'IB3lcPjvWLA',
                 'ext': 'm4a',
-                'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
-                'description': 'md5:1900ed86ee514927b9e00fbead6969a5',
+                'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
+                'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
                 'duration': 244,
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
-                'license': 'Standard YouTube License',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -640,13 +633,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'nfWlot6h_JM',
                 'ext': 'm4a',
                 'title': 'Taylor Swift - Shake It Off',
-                'alt_title': 'Shake It Off',
-                'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
+                'description': 'md5:bec2185232c05479482cb5a9b82719bf',
                 'duration': 242,
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
-                'license': 'Standard YouTube License',
                 'creator': 'Taylor Swift',
             },
             'params': {
@@ -662,10 +653,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'duration': 219,
                 'upload_date': '20100909',
-                'uploader': 'TJ Kirk',
+                'uploader': 'Amazing Atheist',
                 'uploader_id': 'TheAmazingAtheist',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
-                'license': 'Standard YouTube License',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
@@ -683,7 +673,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'WitcherGame',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
-                'license': 'Standard YouTube License',
                 'age_limit': 18,
             },
         },
@@ -692,7 +681,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
             'info_dict': {
                 'id': '6kLq3WMV1nU',
-                'ext': 'webm',
+                'ext': 'mp4',
                 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
                 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
                 'duration': 246,
@@ -700,7 +689,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'LloydVEVO',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
                 'upload_date': '20110629',
-                'license': 'Standard YouTube License',
                 'age_limit': 18,
             },
         },
@@ -718,7 +706,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
-                'license': 'Standard YouTube License',
                 'title': 'Deadmau5 - Some Chords (HD)',
                 'alt_title': 'Some Chords',
             },
@@ -736,7 +723,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
-                'license': 'Standard YouTube License',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympic',
                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
@@ -758,7 +744,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                 'uploader': '孫ᄋᄅ',
-                'license': 'Standard YouTube License',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
             },
         },
@@ -792,7 +777,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'dorappi2000',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
                 'uploader': 'dorappi2000',
-                'license': 'Standard YouTube License',
                 'formats': 'mincount:31',
             },
             'skip': 'not actual anymore',
@@ -808,7 +792,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
-                'license': 'Standard YouTube License',
                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
             },
             'params': {
@@ -881,6 +864,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'This video is not available.',
         },
         {
             # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
@@ -917,7 +901,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'IronSoulElf',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
                 'uploader': 'IronSoulElf',
-                'license': 'Standard YouTube License',
                 'creator': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
                 'track': 'Dark Walk - Position Music',
                 'artist': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
@@ -1021,13 +1004,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'iqKdEhx-dD4',
                 'ext': 'mp4',
                 'title': 'Isolation - Mind Field (Ep 1)',
-                'description': 'md5:25b78d2f64ae81719f5c96319889b736',
+                'description': 'md5:46a29be4ceffa65b92d277b93f463c0f',
                 'duration': 2085,
                 'upload_date': '20170118',
                 'uploader': 'Vsauce',
                 'uploader_id': 'Vsauce',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
-                'license': 'Standard YouTube License',
                 'series': 'Mind Field',
                 'season_number': 1,
                 'episode_number': 1,
@@ -1053,7 +1035,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'New Century Foundation',
                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
-                'license': 'Standard YouTube License',
             },
             'params': {
                 'skip_download': True,
@@ -1081,6 +1062,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # DRM protected
             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
             'only_matching': True,
+        },
+        {
+            # Video with unsupported adaptive stream type formats
+            'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
+            'info_dict': {
+                'id': 'Z4Vy8R84T1U',
+                'ext': 'mp4',
+                'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
+                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+                'duration': 433,
+                'upload_date': '20130923',
+                'uploader': 'Amelia Putri Harwita',
+                'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
+                'formats': 'maxcount:10',
+            },
+            'params': {
+                'skip_download': True,
+                'youtube_include_dash_manifest': False,
+            },
         }
     ]
 

From b0d73a745656443baf3e2134d9ca03bd3a4a934c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Jan 2019 02:20:10 +0700
Subject: [PATCH 321/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3d60754c5..d25169074 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Core
++ [test/helper] Add support for maxcount and count collection len checkers
+* [downloader/hls] Fix uplynk ad skipping (#18824)
+* [postprocessor/ffmpeg] Improve ffmpeg version parsing (#18813)
+
+Extractors
+* [youtube] Skip unsupported adaptive stream type (#18804)
++ [youtube] Extract DASH formats from player response (#18804)
+* [funimation] Fix extraction (#14089)
+* [skylinewebcams] Fix extraction (#18853)
++ [curiositystream] Add support for non app URLs
++ [bitchute] Check formats (#18833)
+* [wistia] Extend URL regular expression (#18823)
++ [playplustv] Add support for playplus.com (#18789)
+
+
 version 2019.01.10
 
 Core

From bfc8eeea57251796d44f1e10b61d06760690bc25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Jan 2019 02:24:08 +0700
Subject: [PATCH 322/510] release 2019.01.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d6abdbcb7..650f78511 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.16*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.10
+[debug] youtube-dl version 2019.01.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index d25169074..13019bf2b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.16
 
 Core
 + [test/helper] Add support for maxcount and count collection len checkers
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5ba61f489..c13f3a38a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.10'
+__version__ = '2019.01.16'

From fa4ac365f69cbd51e4c9801984ebea49a12825b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Jan 2019 10:24:44 +0700
Subject: [PATCH 323/510] [youtube] Extend JS player signature function name
 regexes (closes #18890, closes #18891, closes #18893)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 730935657..c8bf98b58 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1198,8 +1198,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         funcname = self._search_regex(
             (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*c\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*c\s*&&\s*d\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 

From f53cecd796dbb698abbde6ac2f7f973dba78f8a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Jan 2019 10:25:50 +0700
Subject: [PATCH 324/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 13019bf2b..171034a75 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+version <unreleased>
+
+Extractors
+* [youtube] Extend JS player signature function name regular expressions
+  (#18890, #18891, #18893)
+
+
 version 2019.01.16
 
 Core

From 29639b363ddab7903ceae096912a0227c8017533 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Jan 2019 10:27:17 +0700
Subject: [PATCH 325/510] release 2019.01.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 650f78511..841bca914 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.16*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.17*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.16
+[debug] youtube-dl version 2019.01.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 171034a75..902301765 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.17
 
 Extractors
 * [youtube] Extend JS player signature function name regular expressions
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c13f3a38a..ea3f62928 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.16'
+__version__ = '2019.01.17'

From 79fec976b0c250446ea9a9eb7323fb2045ee37fe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 17 Jan 2019 09:44:08 +0100
Subject: [PATCH 326/510] [vimeo] fix extraction for password protected player
 URLs(closes #18889)

---
 youtube_dl/extractor/vimeo.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5e15f060b..fd37f919b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
 import json
 import re
 import itertools
@@ -392,6 +393,22 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://player.vimeo.com/video/68375962',
+            'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
+            'info_dict': {
+                'id': '68375962',
+                'ext': 'mp4',
+                'title': 'youtube-dl password protected test video',
+                'uploader_url': r're:https?://(?:www\.)?vimeo\.com/user18948128',
+                'uploader_id': 'user18948128',
+                'uploader': 'Jaime Marquínez Ferrándiz',
+                'duration': 10,
+            },
+            'params': {
+                'videopassword': 'youtube-dl',
+            },
+        },
         {
             'url': 'http://vimeo.com/moogaloop.swf?clip_id=2539741',
             'only_matching': True,
@@ -452,7 +469,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
-        data = urlencode_postdata({'password': password})
+        data = urlencode_postdata({
+            'password': base64.b64encode(password.encode()),
+        })
         pass_url = url + '/check-password'
         password_request = sanitized_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')

From e2dd132f054df5b6c09b7c274752a77d8ba44f8d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 17 Jan 2019 23:56:37 +0100
Subject: [PATCH 327/510] [cartoonnetwork] fix extraction(closes #15664)(closes
 #17224)

---
 youtube_dl/extractor/cartoonnetwork.py | 56 +++++++++++++++++---------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
index 6aeebd7b3..48b33617f 100644
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -1,20 +1,19 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .turner import TurnerBaseIE
+from ..utils import int_or_none
 
 
 class CartoonNetworkIE(TurnerBaseIE):
     _VALID_URL = r'https?://(?:www\.)?cartoonnetwork\.com/video/(?:[^/]+/)+(?P<id>[^/?#]+)-(?:clip|episode)\.html'
     _TEST = {
-        'url': 'http://www.cartoonnetwork.com/video/teen-titans-go/starfire-the-cat-lady-clip.html',
+        'url': 'https://www.cartoonnetwork.com/video/ben-10/how-to-draw-upgrade-episode.html',
         'info_dict': {
-            'id': '8a250ab04ed07e6c014ef3f1e2f9016c',
+            'id': '6e3375097f63874ebccec7ef677c1c3845fa850e',
             'ext': 'mp4',
-            'title': 'Starfire the Cat Lady',
-            'description': 'Robin decides to become a cat so that Starfire will finally love him.',
+            'title': 'How to Draw Upgrade',
+            'description': 'md5:2061d83776db7e8be4879684eefe8c0f',
         },
         'params': {
             # m3u8 download
@@ -25,18 +24,39 @@ class CartoonNetworkIE(TurnerBaseIE):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        id_type, video_id = re.search(r"_cnglobal\.cvp(Video|Title)Id\s*=\s*'([^']+)';", webpage).groups()
-        query = ('id' if id_type == 'Video' else 'titleId') + '=' + video_id
-        return self._extract_cvp_info(
-            'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?networkName=CN2&' + query, video_id, {
-                'secure': {
-                    'media_src': 'http://androidhls-secure.cdn.turner.com/toon/big',
-                    'tokenizer_src': 'https://token.vgtf.net/token/token_mobile',
-                },
-            }, {
+
+        def find_field(global_re, name, content_re=None, value_re='[^"]+', fatal=False):
+            metadata_re = ''
+            if content_re:
+                metadata_re = r'|video_metadata\.content_' + content_re
+            return self._search_regex(
+                r'(?:_cnglobal\.currentVideo\.%s%s)\s*=\s*"(%s)";' % (global_re, metadata_re, value_re),
+                webpage, name, fatal=fatal)
+
+        media_id = find_field('mediaId', 'media id', 'id', '[0-9a-f]{40}', True)
+        title = find_field('episodeTitle', 'title', '(?:episodeName|name)', fatal=True)
+
+        info = self._extract_ngtv_info(
+            media_id, {'networkId': 'cartoonnetwork'}, {
                 'url': url,
                 'site_name': 'CartoonNetwork',
-                'auth_required': self._search_regex(
-                    r'_cnglobal\.cvpFullOrPreviewAuth\s*=\s*(true|false);',
-                    webpage, 'auth required', default='false') == 'true',
+                'auth_required': find_field('authType', 'auth type') != 'unauth',
             })
+
+        series = find_field(
+            'propertyName', 'series', 'showName') or self._html_search_meta('partOfSeries', webpage)
+        info.update({
+            'id': media_id,
+            'display_id': display_id,
+            'title': title,
+            'description': self._html_search_meta('description', webpage),
+            'series': series,
+            'episode': title,
+        })
+
+        for field in ('season', 'episode'):
+            field_name = field + 'Number'
+            info[field + '_number'] = int_or_none(find_field(
+                field_name, field + ' number', value_re=r'\d+') or self._html_search_meta(field_name, webpage))
+
+        return info

From 2bfc1d9d68dec097fd8093dc0284dd0cd64beb2e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 19 Jan 2019 21:25:15 +0100
Subject: [PATCH 328/510] [extractor/common] imporove HLS video only format
 detection(closes #18923)

---
 youtube_dl/extractor/common.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e7febcad..af621b74b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1596,6 +1596,7 @@ class InfoExtractor(object):
         # References:
         # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
         # 2. https://github.com/rg3/youtube-dl/issues/12211
+        # 3. https://github.com/rg3/youtube-dl/issues/18923
 
         # We should try extracting formats only from master playlists [1, 4.3.4],
         # i.e. playlists that describe available qualities. On the other hand
@@ -1667,11 +1668,16 @@ class InfoExtractor(object):
             rendition = stream_group[0]
             return rendition.get('NAME') or stream_group_id
 
+        # parse EXT-X-MEDIA tags before EXT-X-STREAM-INF inorder to have the
+        # chance to detect video only formats when EXT-X-STREAM-INF tags
+        # precede EXT-X-MEDIA tags in HLS manifest such as [3].
+        for line in m3u8_doc.splitlines():
+            if line.startswith('#EXT-X-MEDIA:'):
+                extract_media(line)
+
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
                 last_stream_inf = parse_m3u8_attributes(line)
-            elif line.startswith('#EXT-X-MEDIA:'):
-                extract_media(line)
             elif line.startswith('#') or not line.strip():
                 continue
             else:

From f28363ad1fb45b4450ae6f09ff9aff8f93227e40 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 19 Jan 2019 21:25:53 +0100
Subject: [PATCH 329/510] [ted] correct acodec for http formats(#18923)

---
 youtube_dl/extractor/ted.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index d3e4205f5..645942dfd 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -265,6 +265,8 @@ class TEDIE(InfoExtractor):
                     'format_id': m3u8_format['format_id'].replace('hls', 'http'),
                     'protocol': 'http',
                 })
+                if f.get('acodec') == 'none':
+                    del f['acodec']
                 formats.append(f)
 
         audio_download = talk_info.get('audioDownload')

From 379306ef55b64c966392c072b17a450831fec252 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 19 Jan 2019 21:35:02 +0100
Subject: [PATCH 330/510] [extractor/common] fix typo

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index af621b74b..6e36e6778 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1668,7 +1668,7 @@ class InfoExtractor(object):
             rendition = stream_group[0]
             return rendition.get('NAME') or stream_group_id
 
-        # parse EXT-X-MEDIA tags before EXT-X-STREAM-INF inorder to have the
+        # parse EXT-X-MEDIA tags before EXT-X-STREAM-INF in order to have the
         # chance to detect video only formats when EXT-X-STREAM-INF tags
         # precede EXT-X-MEDIA tags in HLS manifest such as [3].
         for line in m3u8_doc.splitlines():

From 2cc779f497ae20d6e0e28fc546a25723cfea631a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Jan 2019 13:48:09 +0700
Subject: [PATCH 331/510] [YoutubeDL] Add negation support for string
 comparisons in format selection expressions (closes #18600, closes #18805)

---
 README.md               |  3 ++-
 test/test_YoutubeDL.py  | 46 +++++++++++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py |  9 +++++---
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 70bcfaccf..886696015 100644
--- a/README.md
+++ b/README.md
@@ -667,13 +667,14 @@ The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `
  - `asr`: Audio sampling rate in Hertz
  - `fps`: Frame rate
 
-Also filtering work for comparisons `=` (equals), `!=` (not equals), `^=` (begins with), `$=` (ends with), `*=` (contains) and following string meta fields:
+Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends with), `*=` (contains) and following string meta fields:
  - `ext`: File extension
  - `acodec`: Name of the audio codec in use
  - `vcodec`: Name of the video codec in use
  - `container`: Name of the container format
  - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`)
  - `format_id`: A short description of the format
+Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain).
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the video hoster.
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f0f5a8470..df8994b84 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -239,6 +239,52 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
 
+    def test_format_selection_string_ops(self):
+        formats = [
+            {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
+        ]
+        info_dict = _make_result(formats)
+
+        # equals (=)
+        ydl = YDL({'format': '[format_id=abc-cba]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'abc-cba')
+
+        # does not equal (!=)
+        ydl = YDL({'format': '[format_id!=abc-cba]'})
+        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
+
+        # starts with (^=)
+        ydl = YDL({'format': '[format_id^=abc]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'abc-cba')
+
+        # does not start with (!^=)
+        ydl = YDL({'format': '[format_id!^=abc-cba]'})
+        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
+
+        # ends with ($=)
+        ydl = YDL({'format': '[format_id$=cba]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'abc-cba')
+
+        # does not end with (!$=)
+        ydl = YDL({'format': '[format_id!$=abc-cba]'})
+        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
+
+        # contains (*=)
+        ydl = YDL({'format': '[format_id*=-]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'abc-cba')
+
+        # does not contain (!*=)
+        ydl = YDL({'format': '[format_id!*=-]'})
+        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
+
     def test_youtube_format_selection(self):
         order = [
             '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4493fd0e1..a827414dc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1063,21 +1063,24 @@ class YoutubeDL(object):
         if not m:
             STR_OPERATORS = {
                 '=': operator.eq,
-                '!=': operator.ne,
                 '^=': lambda attr, value: attr.startswith(value),
                 '$=': lambda attr, value: attr.endswith(value),
                 '*=': lambda attr, value: value in attr,
             }
             str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
-                \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
+                \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9._-]+)
                 \s*$
                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
             m = str_operator_rex.search(filter_spec)
             if m:
                 comparison_value = m.group('value')
-                op = STR_OPERATORS[m.group('op')]
+                str_op = STR_OPERATORS[m.group('op')]
+                if m.group('negation'):
+                    op = lambda attr, value: not str_op
+                else:
+                    op = str_op
 
         if not m:
             raise ValueError('Invalid filter specification %r' % filter_spec)

From 4e58d9fabbfef80b2ecc0d20959bdf9dd3705e73 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 20 Jan 2019 14:23:35 +0700
Subject: [PATCH 332/510] [README.md] Fix formatting

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 886696015..4ba982907 100644
--- a/README.md
+++ b/README.md
@@ -674,6 +674,7 @@ Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends
  - `container`: Name of the container format
  - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`)
  - `format_id`: A short description of the format
+
 Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain).
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the video hoster.

From fc746c3fdd7d40935a11e72b5cb69a8aee840e94 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 20 Jan 2019 09:04:51 +0100
Subject: [PATCH 333/510] [test/test_InfoExtractor] add test for #18923

---
 test/test_InfoExtractor.py        | 59 ++++++++++++++++++++++++++++++-
 test/testdata/m3u8/ted_18923.m3u8 | 28 +++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 test/testdata/m3u8/ted_18923.m3u8

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 06be72616..75fa0bbb7 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -497,7 +497,64 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'width': 1280,
                     'height': 720,
                 }]
-            )
+            ),
+            (
+                # https://github.com/rg3/youtube-dl/issues/18923
+                # https://www.ted.com/talks/boris_hesser_a_grassroots_healthcare_revolution_in_africa
+                'ted_18923',
+                'http://hls.ted.com/talks/31241.m3u8',
+                [{
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/audio/600k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '600k-Audio',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/audio/600k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '68',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/64k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '163',
+                    'acodec': 'none',
+                    'width': 320,
+                    'height': 180,
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/180k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '481',
+                    'acodec': 'none',
+                    'width': 512,
+                    'height': 288,
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/320k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '769',
+                    'acodec': 'none',
+                    'width': 512,
+                    'height': 288,
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/450k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '984',
+                    'acodec': 'none',
+                    'width': 512,
+                    'height': 288,
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/600k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '1255',
+                    'acodec': 'none',
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/950k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '1693',
+                    'acodec': 'none',
+                    'width': 853,
+                    'height': 480,
+                }, {
+                    'url': 'http://hls.ted.com/videos/BorisHesser_2018S/video/1500k.m3u8?nobumpers=true&uniqueId=76011e2b',
+                    'format_id': '2462',
+                    'acodec': 'none',
+                    'width': 1280,
+                    'height': 720,
+                }]
+            ),
         ]
 
         for m3u8_file, m3u8_url, expected_formats in _TEST_CASES:
diff --git a/test/testdata/m3u8/ted_18923.m3u8 b/test/testdata/m3u8/ted_18923.m3u8
new file mode 100644
index 000000000..52a27118b
--- /dev/null
+++ b/test/testdata/m3u8/ted_18923.m3u8
@@ -0,0 +1,28 @@
+#EXTM3U
+#EXT-X-VERSION:4
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=1255659,PROGRAM-ID=1,CODECS="avc1.42c01e,mp4a.40.2",RESOLUTION=640x360
+/videos/BorisHesser_2018S/video/600k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=163154,PROGRAM-ID=1,CODECS="avc1.42c00c,mp4a.40.2",RESOLUTION=320x180
+/videos/BorisHesser_2018S/video/64k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=481701,PROGRAM-ID=1,CODECS="avc1.42c015,mp4a.40.2",RESOLUTION=512x288
+/videos/BorisHesser_2018S/video/180k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=769968,PROGRAM-ID=1,CODECS="avc1.42c015,mp4a.40.2",RESOLUTION=512x288
+/videos/BorisHesser_2018S/video/320k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=984037,PROGRAM-ID=1,CODECS="avc1.42c015,mp4a.40.2",RESOLUTION=512x288
+/videos/BorisHesser_2018S/video/450k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=1693925,PROGRAM-ID=1,CODECS="avc1.4d401f,mp4a.40.2",RESOLUTION=853x480
+/videos/BorisHesser_2018S/video/950k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=2462469,PROGRAM-ID=1,CODECS="avc1.640028,mp4a.40.2",RESOLUTION=1280x720
+/videos/BorisHesser_2018S/video/1500k.m3u8?nobumpers=true&uniqueId=76011e2b
+#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=68101,PROGRAM-ID=1,CODECS="mp4a.40.2",DEFAULT=YES
+/videos/BorisHesser_2018S/audio/600k.m3u8?nobumpers=true&uniqueId=76011e2b
+
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=74298,PROGRAM-ID=1,CODECS="avc1.42c00c",RESOLUTION=320x180,URI="/videos/BorisHesser_2018S/video/64k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=216200,PROGRAM-ID=1,CODECS="avc1.42c015",RESOLUTION=512x288,URI="/videos/BorisHesser_2018S/video/180k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=304717,PROGRAM-ID=1,CODECS="avc1.42c015",RESOLUTION=512x288,URI="/videos/BorisHesser_2018S/video/320k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=350933,PROGRAM-ID=1,CODECS="avc1.42c015",RESOLUTION=512x288,URI="/videos/BorisHesser_2018S/video/450k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=495850,PROGRAM-ID=1,CODECS="avc1.42c01e",RESOLUTION=640x360,URI="/videos/BorisHesser_2018S/video/600k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=810750,PROGRAM-ID=1,CODECS="avc1.4d401f",RESOLUTION=853x480,URI="/videos/BorisHesser_2018S/video/950k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=1273700,PROGRAM-ID=1,CODECS="avc1.640028",RESOLUTION=1280x720,URI="/videos/BorisHesser_2018S/video/1500k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b"
+
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="600k",LANGUAGE="en",NAME="Audio",AUTOSELECT=YES,DEFAULT=YES,URI="/videos/BorisHesser_2018S/audio/600k.m3u8?nobumpers=true&uniqueId=76011e2b",BANDWIDTH=614400

From 15870747f02effba9376fd1a1500c32d36d1b811 Mon Sep 17 00:00:00 2001
From: aviperes <avipr24@gmail.com>
Date: Sun, 20 Jan 2019 10:15:01 +0200
Subject: [PATCH 334/510] [odnoklassniki] Detect paid videos

---
 youtube_dl/extractor/odnoklassniki.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 190d8af4d..114b93c07 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -115,6 +115,10 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'https://m.ok.ru/dk?st.cmd=movieLayer&st.discId=863789452017&st.retLoc=friend&st.rtu=%2Fdk%3Fst.cmd%3DfriendMovies%26st.mode%3Down%26st.mrkId%3D%257B%2522uploadedMovieMarker%2522%253A%257B%2522marker%2522%253A%25221519410114503%2522%252C%2522hasMore%2522%253Atrue%257D%252C%2522sharedMovieMarker%2522%253A%257B%2522marker%2522%253Anull%252C%2522hasMore%2522%253Afalse%257D%257D%26st.friendId%3D561722190321%26st.frwd%3Don%26_prevCmd%3DfriendMovies%26tkn%3D7257&st.discType=MOVIE&st.mvId=863789452017&_prevCmd=friendMovies&tkn=3648#lst#',
         'only_matching': True,
+    }, {
+        # Paid video
+        'url': 'https://ok.ru/video/954886983203',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -244,6 +248,11 @@ class OdnoklassnikiIE(InfoExtractor):
                 'ext': 'flv',
             })
 
+        if not formats:
+            payment_info = metadata.get('paymentInfo')
+            if payment_info:
+                raise ExtractorError('This video is paid, subscribe to download it', expected=True)
+
         self._sort_formats(formats)
 
         info['formats'] = formats

From 31fbedc06a349bc555ab934588544f75734e3a55 Mon Sep 17 00:00:00 2001
From: jhwgh1968 <jhwgh1968@users.noreply.github.com>
Date: Sun, 20 Jan 2019 09:10:46 +0000
Subject: [PATCH 335/510] [instagram] Add base extractor for playlists and tag
 extractor

---
 youtube_dl/extractor/extractors.py |   6 +-
 youtube_dl/extractor/instagram.py  | 136 +++++++++++++++++++++--------
 2 files changed, 105 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index de38c6641..24361def4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -494,7 +494,11 @@ from .ina import InaIE
 from .inc import IncIE
 from .indavideo import IndavideoEmbedIE
 from .infoq import InfoQIE
-from .instagram import InstagramIE, InstagramUserIE
+from .instagram import (
+    InstagramIE,
+    InstagramUserIE,
+    InstagramTagIE,
+)
 from .internazionale import InternazionaleIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 7e0e838f0..ffd87b55f 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -227,44 +227,37 @@ class InstagramIE(InfoExtractor):
         }
 
 
-class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
-    IE_DESC = 'Instagram user profile'
-    IE_NAME = 'instagram:user'
-    _TEST = {
-        'url': 'https://instagram.com/porsche',
-        'info_dict': {
-            'id': 'porsche',
-            'title': 'porsche',
-        },
-        'playlist_count': 5,
-        'params': {
-            'extract_flat': True,
-            'skip_download': True,
-            'playlistend': 5,
-        }
-    }
+class InstagramPlaylistIE(InfoExtractor):
+    # A superclass for handling any kind of query based on GraphQL which
+    # results in a playlist.
 
-    _gis_tmpl = None
+    _gis_tmpl = None  # used to cache GIS request type
 
-    def _entries(self, data):
+    def _parse_graphql(self, webpage, item_id):
+        # Reads a webpage and returns its GraphQL data.
+        return self._parse_json(
+            self._search_regex(
+                r'sharedData\s*=\s*({.+?})\s*;\s*[<\n]', webpage, 'data'),
+            item_id)
+
+    def _extract_graphql(self, data, url):
+        # Parses GraphQL queries containing videos and generates a playlist.
         def get_count(suffix):
             return int_or_none(try_get(
                 node, lambda x: x['edge_media_' + suffix]['count']))
 
-        uploader_id = data['entry_data']['ProfilePage'][0]['graphql']['user']['id']
+        uploader_id = self._match_id(url)
         csrf_token = data['config']['csrf_token']
         rhx_gis = data.get('rhx_gis') or '3c7ca9dcefcf966d11dacf1f151335e8'
 
-        self._set_cookie('instagram.com', 'ig_pr', '1')
-
         cursor = ''
         for page_num in itertools.count(1):
-            variables = json.dumps({
-                'id': uploader_id,
+            variables = {
                 'first': 12,
                 'after': cursor,
-            })
+            }
+            variables.update(self._query_vars_for(data))
+            variables = json.dumps(variables)
 
             if self._gis_tmpl:
                 gis_tmpls = [self._gis_tmpl]
@@ -276,21 +269,26 @@ class InstagramUserIE(InfoExtractor):
                     '%s:%s:%s' % (rhx_gis, csrf_token, std_headers['User-Agent']),
                 ]
 
+            # try all of the ways to generate a GIS query, and not only use the
+            # first one that works, but cache it for future requests
             for gis_tmpl in gis_tmpls:
                 try:
-                    media = self._download_json(
+                    json_data = self._download_json(
                         'https://www.instagram.com/graphql/query/', uploader_id,
                         'Downloading JSON page %d' % page_num, headers={
                             'X-Requested-With': 'XMLHttpRequest',
                             'X-Instagram-GIS': hashlib.md5(
                                 ('%s:%s' % (gis_tmpl, variables)).encode('utf-8')).hexdigest(),
                         }, query={
-                            'query_hash': '42323d64886122307be10013ad2dcc44',
+                            'query_hash': self._QUERY_HASH,
                             'variables': variables,
-                        })['data']['user']['edge_owner_to_timeline_media']
+                        })
+                    media = self._parse_timeline_from(json_data)
                     self._gis_tmpl = gis_tmpl
                     break
                 except ExtractorError as e:
+                    # if it's an error caused by a bad query, and there are
+                    # more GIS templates to try, ignore it and keep trying
                     if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                         if gis_tmpl != gis_tmpls[-1]:
                             continue
@@ -348,14 +346,80 @@ class InstagramUserIE(InfoExtractor):
                 break
 
     def _real_extract(self, url):
-        username = self._match_id(url)
+        user_or_tag = self._match_id(url)
+        webpage = self._download_webpage(url, user_or_tag)
+        data = self._parse_graphql(webpage, user_or_tag)
 
-        webpage = self._download_webpage(url, username)
-
-        data = self._parse_json(
-            self._search_regex(
-                r'sharedData\s*=\s*({.+?})\s*;\s*[<\n]', webpage, 'data'),
-            username)
+        self._set_cookie('instagram.com', 'ig_pr', '1')
 
         return self.playlist_result(
-            self._entries(data), username, username)
+            self._extract_graphql(data, url), user_or_tag, user_or_tag)
+
+
+class InstagramUserIE(InstagramPlaylistIE):
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
+    IE_DESC = 'Instagram user profile'
+    IE_NAME = 'instagram:user'
+    _TEST = {
+        'url': 'https://instagram.com/porsche',
+        'info_dict': {
+            'id': 'porsche',
+            'title': 'porsche',
+        },
+        'playlist_count': 5,
+        'params': {
+            'extract_flat': True,
+            'skip_download': True,
+            'playlistend': 5,
+        }
+    }
+
+    _QUERY_HASH = '42323d64886122307be10013ad2dcc44',
+
+    @staticmethod
+    def _parse_timeline_from(data):
+        # extracts the media timeline data from a GraphQL result
+        return data['data']['user']['edge_owner_to_timeline_media']
+
+    @staticmethod
+    def _query_vars_for(data):
+        # returns a dictionary of variables to add to the timeline query based
+        # on the GraphQL of the original page
+        return {
+            'id': data['entry_data']['ProfilePage'][0]['graphql']['user']['id']
+        }
+
+
+class InstagramTagIE(InstagramPlaylistIE):
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/explore/tags/(?P<id>[^/]+)'
+    IE_DESC = 'Instagram hashtag search'
+    IE_NAME = 'instagram:tag'
+    _TEST = {
+        'url': 'https://instagram.com/explore/tags/lolcats',
+        'info_dict': {
+            'id': 'lolcats',
+            'title': 'lolcats',
+        },
+        'playlist_count': 50,
+        'params': {
+            'extract_flat': True,
+            'skip_download': True,
+            'playlistend': 50,
+        }
+    }
+
+    _QUERY_HASH = 'f92f56d47dc7a55b606908374b43a314',
+
+    @staticmethod
+    def _parse_timeline_from(data):
+        # extracts the media timeline data from a GraphQL result
+        return data['data']['hashtag']['edge_hashtag_to_media']
+
+    @staticmethod
+    def _query_vars_for(data):
+        # returns a dictionary of variables to add to the timeline query based
+        # on the GraphQL of the original page
+        return {
+            'tag_name':
+                data['entry_data']['TagPage'][0]['graphql']['hashtag']['name']
+        }

From 6ca3fa898cd066422daea3d5be53efdae22187d8 Mon Sep 17 00:00:00 2001
From: yonaikerlol <lawlietrs7@gmail.com>
Date: Sun, 20 Jan 2019 05:24:21 -0400
Subject: [PATCH 336/510] [streamango] Add support for fruithosts.net

---
 youtube_dl/extractor/streamango.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index fcaa5ac0b..efb259f96 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class StreamangoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:streamango\.com|fruithosts\.net)/(?:f|embed)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://streamango.com/f/clapasobsptpkdfe/20170315_150006_mp4',
         'md5': 'e992787515a182f55e38fc97588d802a',
@@ -38,6 +38,9 @@ class StreamangoIE(InfoExtractor):
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
         'only_matching': True,
+    }, {
+        'url': 'https://fruithosts.net/f/mreodparcdcmspsm/w1f1_r4lph_2018_brrs_720p_latino_mp4',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 289ef490f77cb35c43d98b9d2ea4cf529c24895e Mon Sep 17 00:00:00 2001
From: Anthony Fok <foka@debian.org>
Date: Sun, 30 Dec 2018 02:44:40 -0700
Subject: [PATCH 337/510] [hketv] Add extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/hketv.py      | 185 +++++++++++++++++++++++++++++
 2 files changed, 186 insertions(+)
 create mode 100644 youtube_dl/extractor/hketv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 24361def4..574a47e6d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -452,6 +452,7 @@ from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .hgtv import HGTVComShowIE
+from .hketv import HKETVIE
 from .hidive import HiDiveIE
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
diff --git a/youtube_dl/extractor/hketv.py b/youtube_dl/extractor/hketv.py
new file mode 100644
index 000000000..b5790cdee
--- /dev/null
+++ b/youtube_dl/extractor/hketv.py
@@ -0,0 +1,185 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    int_or_none,
+    merge_dicts,
+    str_or_none,
+    str_to_int,
+    try_get,
+    unified_strdate,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class HKETVIE(InfoExtractor):
+    IE_NAME = 'hketv'
+    IE_DESC = '香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau'
+    _GEO_BYPASS = False
+    _GEO_COUNTRIES = ['HK']
+    _VALID_URL = r'https?://(?:www\.)?hkedcity\.net/etv/resource/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://www.hkedcity.net/etv/resource/2932360618',
+        'md5': 'f193712f5f7abb208ddef3c5ea6ed0b7',
+        'info_dict': {
+            'id': '2932360618',
+            'ext': 'mp4',
+            'title': '喜閱一生(共享閱讀樂) (中、英文字幕可供選擇)',
+            'description': '本節目輯錄了「閱讀滿Fun嘉年華」和「二○一八響應世界閱讀日――悅愛閱讀・愈讀愈愛」的活動花絮，並由學者、作家、演藝界人士等，分享培養子女閱讀興趣和習慣的方法，以及呼籲大家一同分享閱讀的樂趣。',
+            'upload_date': '20181024',
+            'duration': 900,
+            'subtitles': {
+                'en': [{
+                    'url': 'https://apps.hkedcity.net/media/mediaplayer/caption.php?f=74395&lang=en',
+                    'ext': 'srt',
+                }],
+                'zh-Hant': [{
+                    'url': 'https://apps.hkedcity.net/media/mediaplayer/caption.php?f=74395&lang=qmt',
+                    'ext': 'srt',
+                }],
+            }
+        },
+    }, {
+        'url': 'https://www.hkedcity.net/etv/resource/972641418',
+        'md5': '1ed494c1c6cf7866a8290edad9b07dc9',
+        'info_dict': {
+            'id': '972641418',
+            'ext': 'mp4',
+            'title': '衣冠楚楚 (天使系列之一)',
+            'description': '天國仙境，有兩位可愛的天使小姐妹。她們對幾千年來天使衣著一成不變頗有不滿。她們下望人世間：只見人們穿著七彩繽紛、款式各異的服裝，漂亮極了。天使姐妹決定下凡考察衣著，以設計天使新裝。  下到人間，姐妹試穿各式各樣的衣著，引發連串奇特有趣的情節：她們穿著校服在街上閒逛時，被女警誤認為逃學而送回學校，到校後又被體育老師誤認為是新同學，匆匆忙忙換上運動服後在操場上大顯神通。她們穿著護士服在醫院散步時，又被誤認為當班護士，而投入追尋失蹤病童、治病救人的工作中去。姐妹倆還到過玩具店，與布娃娃們談論衣著。她們也去過服裝設計學校，被當成小模特兒而試穿各式服裝。最令姐妹倆興奮的是一場盛大的民族服裝表演會。身穿盛裝的十二個民族的少女在台上翩翩起舞，各種服飾七彩繽紛、美不勝收。姐妹們情不自禁地穿上民族服裝，小天使變成了少數民族姑娘……最後天使姐妹回到天上，對於天使究竟穿甚麼樣的衣服好，她們還是拿不定主意。  節目通過天使姐妹的奇特經歷，反復示範各式衣服鞋襪的正確讀音及談論衣著時的常用句式，並以盛大的民族服裝表演活動，帶出有關服裝的文化知識。內容豐富而饒有趣味。',
+            'upload_date': '20070109',
+            'duration': 907,
+            'subtitles': {},
+        },
+    }]
+
+    _CC_LANGS = {
+        '中文（繁體中文）': 'zh-Hant',
+        '中文（简体中文）': 'zh-Hans',
+        'English': 'en',
+        'Bahasa Indonesia': 'id',
+        '\u0939\u093f\u0928\u094d\u0926\u0940': 'hi',
+        '\u0928\u0947\u092a\u093e\u0932\u0940': 'ne',
+        'Tagalog': 'tl',
+        '\u0e44\u0e17\u0e22': 'th',
+        '\u0627\u0631\u062f\u0648': 'ur',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_meta('ed_title', webpage, fatal=True)
+
+        file_id = self._html_search_regex(r'post_var\["file_id"\]\s*=\s*(.+?);', webpage, 'file ID')
+        curr_url = self._html_search_regex(r'post_var\["curr_url"\]\s*=\s*"(.+?)";', webpage, 'curr URL')
+        data = {
+            'action': 'get_info',
+            'curr_url': curr_url,
+            'file_id': file_id,
+            'video_url': file_id,
+        }
+        _APPS_BASE_URL = 'https://apps.hkedcity.net'
+        handler_url = _APPS_BASE_URL + '/media/play/handler.php'
+
+        response = self._download_json(
+            handler_url, video_id,
+            data=urlencode_postdata(data),
+            headers=merge_dicts({'Content-Type': 'application/x-www-form-urlencoded'},
+                                self.geo_verification_headers()))
+
+        result = response['result']
+
+        formats = []
+        subtitles = {}
+
+        if response.get('success') and response.get('access'):
+            width = int_or_none(result.get('width'))
+            height = int_or_none(result.get('height'))
+
+            playlist0 = try_get(result, lambda x: x['playlist'][0], dict)
+            fmts = playlist0.get('sources')
+            for fmt in fmts:
+                file_path = fmt.get('file')
+                if file_path:
+                    file_url = urljoin(_APPS_BASE_URL, file_path)
+                    # If we ever wanted to provide the final resolved URL that
+                    # does not require cookies, albeit with a shorter lifespan:
+                    #     urlh = self._downloader.urlopen(file_url)
+                    #     resolved_url = urlh.geturl()
+
+                    label = fmt.get('label')
+                    w = None
+                    h = None
+                    if label == 'HD':
+                        h = 720
+                    elif label == 'SD':
+                        h = 360
+                    if h:
+                        if width and height:
+                            w = h * width // height
+                        else:
+                            w = h * 4 // 3
+
+                    formats.append({
+                        'format_id': label,
+                        'ext': fmt.get('type'),
+                        'url': file_url,
+                        'width': w,
+                        'height': h,
+                    })
+
+            tracks = playlist0.get('tracks', [])
+            for track in tracks:
+                if not isinstance(track, dict):
+                    continue
+                track_kind = str_or_none(track.get('kind'))
+                if not track_kind or not isinstance(track_kind, compat_str):
+                    continue
+                if track_kind.lower() not in ('captions', 'subtitles'):
+                    continue
+                track_url = urljoin(_APPS_BASE_URL, track.get('file'))
+                if not track_url:
+                    continue
+                track_label = track.get('label')
+                subtitles.setdefault(self._CC_LANGS.get(track_label, track_label), []).append({
+                    'url': self._proto_relative_url(track_url),
+                    'ext': 'srt',
+                })
+
+        else:
+            error = clean_html(response.get('access_err_msg'))
+            if 'Video streaming is not available in your country' in error:
+                self.raise_geo_restricted(msg=error, countries=self._GEO_COUNTRIES)
+            else:
+                raise ExtractorError(error)
+
+        # Likes
+        emotion = self._download_json(
+            'https://emocounter.hkedcity.net/handler.php',
+            video_id,
+            data=urlencode_postdata({
+                'action': 'get_emotion',
+                'data[bucket_id]': 'etv',
+                'data[identifier]': video_id,
+            }),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'},
+            fatal=False)
+        like_count = int_or_none(try_get(emotion, lambda x: x['data']['emotion_data'][0]['count']))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._html_search_meta('description', webpage, fatal=False),
+            'upload_date': unified_strdate(self._html_search_meta('ed_date', webpage, fatal=False), day_first=False),
+            'duration': int_or_none(result.get('length')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnail': urljoin(_APPS_BASE_URL, result.get('image')),
+            'view_count': str_to_int(result.get('view_count')),
+            'like_count': like_count,
+        }

From 73c19aaa9f74e9383a7aaf0dfb3c608727d5b6b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Jan 2019 17:42:25 +0700
Subject: [PATCH 338/510] [hketv] Improve and simplify (closes #18696)

---
 youtube_dl/extractor/hketv.py | 180 ++++++++++++++++++----------------
 1 file changed, 93 insertions(+), 87 deletions(-)

diff --git a/youtube_dl/extractor/hketv.py b/youtube_dl/extractor/hketv.py
index b5790cdee..b57927fc1 100644
--- a/youtube_dl/extractor/hketv.py
+++ b/youtube_dl/extractor/hketv.py
@@ -8,8 +8,8 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     merge_dicts,
+    parse_count,
     str_or_none,
-    str_to_int,
     try_get,
     unified_strdate,
     urlencode_postdata,
@@ -30,20 +30,12 @@ class HKETVIE(InfoExtractor):
             'id': '2932360618',
             'ext': 'mp4',
             'title': '喜閱一生(共享閱讀樂) (中、英文字幕可供選擇)',
-            'description': '本節目輯錄了「閱讀滿Fun嘉年華」和「二○一八響應世界閱讀日――悅愛閱讀・愈讀愈愛」的活動花絮，並由學者、作家、演藝界人士等，分享培養子女閱讀興趣和習慣的方法，以及呼籲大家一同分享閱讀的樂趣。',
+            'description': 'md5:d5286d05219ef50e0613311cbe96e560',
             'upload_date': '20181024',
             'duration': 900,
-            'subtitles': {
-                'en': [{
-                    'url': 'https://apps.hkedcity.net/media/mediaplayer/caption.php?f=74395&lang=en',
-                    'ext': 'srt',
-                }],
-                'zh-Hant': [{
-                    'url': 'https://apps.hkedcity.net/media/mediaplayer/caption.php?f=74395&lang=qmt',
-                    'ext': 'srt',
-                }],
-            }
+            'subtitles': 'count:2',
         },
+        'skip': 'Geo restricted to HK',
     }, {
         'url': 'https://www.hkedcity.net/etv/resource/972641418',
         'md5': '1ed494c1c6cf7866a8290edad9b07dc9',
@@ -51,11 +43,15 @@ class HKETVIE(InfoExtractor):
             'id': '972641418',
             'ext': 'mp4',
             'title': '衣冠楚楚 (天使系列之一)',
-            'description': '天國仙境，有兩位可愛的天使小姐妹。她們對幾千年來天使衣著一成不變頗有不滿。她們下望人世間：只見人們穿著七彩繽紛、款式各異的服裝，漂亮極了。天使姐妹決定下凡考察衣著，以設計天使新裝。  下到人間，姐妹試穿各式各樣的衣著，引發連串奇特有趣的情節：她們穿著校服在街上閒逛時，被女警誤認為逃學而送回學校，到校後又被體育老師誤認為是新同學，匆匆忙忙換上運動服後在操場上大顯神通。她們穿著護士服在醫院散步時，又被誤認為當班護士，而投入追尋失蹤病童、治病救人的工作中去。姐妹倆還到過玩具店，與布娃娃們談論衣著。她們也去過服裝設計學校，被當成小模特兒而試穿各式服裝。最令姐妹倆興奮的是一場盛大的民族服裝表演會。身穿盛裝的十二個民族的少女在台上翩翩起舞，各種服飾七彩繽紛、美不勝收。姐妹們情不自禁地穿上民族服裝，小天使變成了少數民族姑娘……最後天使姐妹回到天上，對於天使究竟穿甚麼樣的衣服好，她們還是拿不定主意。  節目通過天使姐妹的奇特經歷，反復示範各式衣服鞋襪的正確讀音及談論衣著時的常用句式，並以盛大的民族服裝表演活動，帶出有關服裝的文化知識。內容豐富而饒有趣味。',
+            'description': 'md5:10bb3d659421e74f58e5db5691627b0f',
             'upload_date': '20070109',
             'duration': 907,
             'subtitles': {},
         },
+        'params': {
+            'geo_verification_proxy': '<HK proxy here>',
+        },
+        'skip': 'Geo restricted to HK',
     }]
 
     _CC_LANGS = {
@@ -69,117 +65,127 @@ class HKETVIE(InfoExtractor):
         '\u0e44\u0e17\u0e22': 'th',
         '\u0627\u0631\u062f\u0648': 'ur',
     }
+    _FORMAT_HEIGHTS = {
+        'SD': 360,
+        'HD': 720,
+    }
+    _APPS_BASE_URL = 'https://apps.hkedcity.net'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_meta('ed_title', webpage, fatal=True)
 
-        file_id = self._html_search_regex(r'post_var\["file_id"\]\s*=\s*(.+?);', webpage, 'file ID')
-        curr_url = self._html_search_regex(r'post_var\["curr_url"\]\s*=\s*"(.+?)";', webpage, 'curr URL')
+        title = (
+            self._html_search_meta(
+                ('ed_title', 'search.ed_title'), webpage, default=None) or
+            self._search_regex(
+                r'data-favorite_title_(?:eng|chi)=(["\'])(?P<id>(?:(?!\1).)+)\1',
+                webpage, 'title', default=None, group='url') or
+            self._html_search_regex(
+                r'<h1>([^<]+)</h1>', webpage, 'title', default=None) or
+            self._og_search_title(webpage)
+        )
+
+        file_id = self._search_regex(
+            r'post_var\[["\']file_id["\']\s*\]\s*=\s*(.+?);',
+            webpage, 'file ID')
+        curr_url = self._search_regex(
+            r'post_var\[["\']curr_url["\']\s*\]\s*=\s*"(.+?)";',
+            webpage, 'curr URL')
         data = {
             'action': 'get_info',
             'curr_url': curr_url,
             'file_id': file_id,
             'video_url': file_id,
         }
-        _APPS_BASE_URL = 'https://apps.hkedcity.net'
-        handler_url = _APPS_BASE_URL + '/media/play/handler.php'
 
         response = self._download_json(
-            handler_url, video_id,
+            self._APPS_BASE_URL + '/media/play/handler.php', video_id,
             data=urlencode_postdata(data),
-            headers=merge_dicts({'Content-Type': 'application/x-www-form-urlencoded'},
-                                self.geo_verification_headers()))
+            headers=merge_dicts({
+                'Content-Type': 'application/x-www-form-urlencoded'},
+                self.geo_verification_headers()))
 
         result = response['result']
 
+        if not response.get('success') or not response.get('access'):
+            error = clean_html(response.get('access_err_msg'))
+            if 'Video streaming is not available in your country' in error:
+                self.raise_geo_restricted(
+                    msg=error, countries=self._GEO_COUNTRIES)
+            else:
+                raise ExtractorError(error, expected=True)
+
         formats = []
+
+        width = int_or_none(result.get('width'))
+        height = int_or_none(result.get('height'))
+
+        playlist0 = result['playlist'][0]
+        for fmt in playlist0['sources']:
+            file_url = urljoin(self._APPS_BASE_URL, fmt.get('file'))
+            if not file_url:
+                continue
+            # If we ever wanted to provide the final resolved URL that
+            # does not require cookies, albeit with a shorter lifespan:
+            #     urlh = self._downloader.urlopen(file_url)
+            #     resolved_url = urlh.geturl()
+            label = fmt.get('label')
+            h = self._FORMAT_HEIGHTS.get(label)
+            w = h * width // height if h and width and height else None
+            formats.append({
+                'format_id': label,
+                'ext': fmt.get('type'),
+                'url': file_url,
+                'width': w,
+                'height': h,
+            })
+        self._sort_formats(formats)
+
         subtitles = {}
-
-        if response.get('success') and response.get('access'):
-            width = int_or_none(result.get('width'))
-            height = int_or_none(result.get('height'))
-
-            playlist0 = try_get(result, lambda x: x['playlist'][0], dict)
-            fmts = playlist0.get('sources')
-            for fmt in fmts:
-                file_path = fmt.get('file')
-                if file_path:
-                    file_url = urljoin(_APPS_BASE_URL, file_path)
-                    # If we ever wanted to provide the final resolved URL that
-                    # does not require cookies, albeit with a shorter lifespan:
-                    #     urlh = self._downloader.urlopen(file_url)
-                    #     resolved_url = urlh.geturl()
-
-                    label = fmt.get('label')
-                    w = None
-                    h = None
-                    if label == 'HD':
-                        h = 720
-                    elif label == 'SD':
-                        h = 360
-                    if h:
-                        if width and height:
-                            w = h * width // height
-                        else:
-                            w = h * 4 // 3
-
-                    formats.append({
-                        'format_id': label,
-                        'ext': fmt.get('type'),
-                        'url': file_url,
-                        'width': w,
-                        'height': h,
-                    })
-
-            tracks = playlist0.get('tracks', [])
-            for track in tracks:
-                if not isinstance(track, dict):
-                    continue
-                track_kind = str_or_none(track.get('kind'))
-                if not track_kind or not isinstance(track_kind, compat_str):
-                    continue
-                if track_kind.lower() not in ('captions', 'subtitles'):
-                    continue
-                track_url = urljoin(_APPS_BASE_URL, track.get('file'))
-                if not track_url:
-                    continue
-                track_label = track.get('label')
-                subtitles.setdefault(self._CC_LANGS.get(track_label, track_label), []).append({
+        tracks = try_get(playlist0, lambda x: x['tracks'], list) or []
+        for track in tracks:
+            if not isinstance(track, dict):
+                continue
+            track_kind = str_or_none(track.get('kind'))
+            if not track_kind or not isinstance(track_kind, compat_str):
+                continue
+            if track_kind.lower() not in ('captions', 'subtitles'):
+                continue
+            track_url = urljoin(self._APPS_BASE_URL, track.get('file'))
+            if not track_url:
+                continue
+            track_label = track.get('label')
+            subtitles.setdefault(self._CC_LANGS.get(
+                track_label, track_label), []).append({
                     'url': self._proto_relative_url(track_url),
                     'ext': 'srt',
                 })
 
-        else:
-            error = clean_html(response.get('access_err_msg'))
-            if 'Video streaming is not available in your country' in error:
-                self.raise_geo_restricted(msg=error, countries=self._GEO_COUNTRIES)
-            else:
-                raise ExtractorError(error)
-
         # Likes
         emotion = self._download_json(
-            'https://emocounter.hkedcity.net/handler.php',
-            video_id,
+            'https://emocounter.hkedcity.net/handler.php', video_id,
             data=urlencode_postdata({
                 'action': 'get_emotion',
                 'data[bucket_id]': 'etv',
                 'data[identifier]': video_id,
             }),
             headers={'Content-Type': 'application/x-www-form-urlencoded'},
-            fatal=False)
-        like_count = int_or_none(try_get(emotion, lambda x: x['data']['emotion_data'][0]['count']))
+            fatal=False) or {}
+        like_count = int_or_none(try_get(
+            emotion, lambda x: x['data']['emotion_data'][0]['count']))
 
         return {
             'id': video_id,
             'title': title,
-            'description': self._html_search_meta('description', webpage, fatal=False),
-            'upload_date': unified_strdate(self._html_search_meta('ed_date', webpage, fatal=False), day_first=False),
+            'description': self._html_search_meta(
+                'description', webpage, fatal=False),
+            'upload_date': unified_strdate(self._html_search_meta(
+                'ed_date', webpage, fatal=False), day_first=False),
             'duration': int_or_none(result.get('length')),
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnail': urljoin(_APPS_BASE_URL, result.get('image')),
-            'view_count': str_to_int(result.get('view_count')),
+            'thumbnail': urljoin(self._APPS_BASE_URL, result.get('image')),
+            'view_count': parse_count(result.get('view_count')),
             'like_count': like_count,
         }

From a1a460759815414c6194bc921ac77a5533b6e02e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Jan 2019 18:21:31 +0700
Subject: [PATCH 339/510] [vimeo] Fix video password verification for videos
 protected by Referer HTTP header

---
 youtube_dl/extractor/vimeo.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fd37f919b..6215b3258 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -435,6 +435,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/160743502/abd0e13fb4',
             'only_matching': True,
         }
+        # https://gettingthingsdone.com/workflowmap/
+        # vimeo embed with check-password page protected by Referer header
     ]
 
     @staticmethod
@@ -465,20 +467,22 @@ class VimeoIE(VimeoBaseInfoExtractor):
         urls = VimeoIE._extract_urls(url, webpage)
         return urls[0] if urls else None
 
-    def _verify_player_video_password(self, url, video_id):
+    def _verify_player_video_password(self, url, video_id, headers):
         password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata({
             'password': base64.b64encode(password.encode()),
         })
-        pass_url = url + '/check-password'
-        password_request = sanitized_Request(pass_url, data)
-        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Referer', url)
-        return self._download_json(
-            password_request, video_id,
-            'Verifying the password', 'Wrong password')
+        headers = merge_dicts(headers, {
+            'Content-Type': 'application/x-www-form-urlencoded',
+        })
+        checked = self._download_json(
+            url + '/check-password', video_id,
+            'Verifying the password', data=data, headers=headers)
+        if checked is False:
+            raise ExtractorError('Wrong video password', expected=True)
+        return checked
 
     def _real_initialize(self):
         self._login()
@@ -591,7 +595,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                                      cause=e)
         else:
             if config.get('view') == 4:
-                config = self._verify_player_video_password(redirect_url, video_id)
+                config = self._verify_player_video_password(redirect_url, video_id, headers)
 
         vod = config.get('video', {}).get('vod', {})
 

From 29cfcb43da8ac60e6c2eddad095a41c800d4d95a Mon Sep 17 00:00:00 2001
From: Alexandre Huot <alexandre.huot@usherbrooke.ca>
Date: Sun, 20 Jan 2019 06:33:09 -0500
Subject: [PATCH 340/510] [radiocanada] Relax DRM check

---
 youtube_dl/extractor/radiocanada.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index b952e59b4..302f67d96 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -49,6 +49,16 @@ class RadioCanadaIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+        },
+        {
+            # with protectionType but not actually DRM protected
+            'url': 'radiocanada:toutv:140872',
+            'info_dict': {
+                'id': '140872',
+                'title': 'Épisode 1',
+                'series': 'District 31',
+            },
+            'only_matching': True,
         }
     ]
 
@@ -67,8 +77,10 @@ class RadioCanadaIE(InfoExtractor):
             el = find_xpath_attr(metadata, './/Meta', 'name', name)
             return el.text if el is not None else None
 
+        # protectionType does not necessarily mean the video is DRM protected (see
+        # https://github.com/rg3/youtube-dl/pull/18609).
         if get_meta('protectionType'):
-            raise ExtractorError('This video is DRM protected.', expected=True)
+            self.report_warning('This video is probably DRM protected.')
 
         device_types = ['ipad']
         if not smuggled_data:

From 6945b9e78f38284eb4e440b7badea2fc60b66c2f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 20 Jan 2019 13:31:41 +0100
Subject: [PATCH 341/510] [extractor/common] improve jwplayer relative url
 handling(closes #18892)

---
 youtube_dl/extractor/common.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6e36e6778..95456b291 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2630,7 +2630,7 @@ class InfoExtractor(object):
                 'id': this_video_id,
                 'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
                 'description': video_data.get('description'),
-                'thumbnail': self._proto_relative_url(video_data.get('image')),
+                'thumbnail': urljoin(base_url, self._proto_relative_url(video_data.get('image'))),
                 'timestamp': int_or_none(video_data.get('pubdate')),
                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
                 'subtitles': subtitles,
@@ -2657,12 +2657,9 @@ class InfoExtractor(object):
         for source in jwplayer_sources_data:
             if not isinstance(source, dict):
                 continue
-            source_url = self._proto_relative_url(source.get('file'))
-            if not source_url:
-                continue
-            if base_url:
-                source_url = compat_urlparse.urljoin(base_url, source_url)
-            if source_url in urls:
+            source_url = urljoin(
+                base_url, self._proto_relative_url(source.get('file')))
+            if not source_url or source_url in urls:
                 continue
             urls.append(source_url)
             source_type = source.get('type') or ''

From fad4ceb53404227f471af2f3544c4c14a5df4acb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Jan 2019 20:21:24 +0700
Subject: [PATCH 342/510] [utils] Fix urljoin for paths with non-http(s)
 schemes

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9e28e008f..409482c3b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -507,6 +507,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(urljoin('http://foo.de/', ''), None)
         self.assertEqual(urljoin('http://foo.de/', ['foobar']), None)
         self.assertEqual(urljoin('http://foo.de/a/b/c.txt', '.././../d.txt'), 'http://foo.de/d.txt')
+        self.assertEqual(urljoin('http://foo.de/a/b/c.txt', 'rtmp://foo.de'), 'rtmp://foo.de')
+        self.assertEqual(urljoin(None, 'rtmp://foo.de'), 'rtmp://foo.de')
 
     def test_url_or_none(self):
         self.assertEqual(url_or_none(None), None)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d2d3c1a9f..d0cb65814 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1868,7 +1868,7 @@ def urljoin(base, path):
         path = path.decode('utf-8')
     if not isinstance(path, compat_str) or not path:
         return None
-    if re.match(r'^(?:https?:)?//', path):
+    if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
         return path
     if isinstance(base, bytes):
         base = base.decode('utf-8')

From 07f9febc4b86a9cd819329f3a7daafdbe9455f40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Jan 2019 22:07:01 +0700
Subject: [PATCH 343/510] [tnaflix] Pass Referer in metadata request (closes
 #18925)

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 6798ef4c3..b3573c6e0 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -96,7 +96,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, 'Downloading metadata',
-            transform_source=fix_xml_ampersands)
+            transform_source=fix_xml_ampersands, headers={'Referer': url})
 
         formats = []
 

From 19d6991312405f5af108af28b3721966720fc72d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Jan 2019 03:03:53 +0700
Subject: [PATCH 344/510] [videomore] Improve extraction and fix season
 extractor (closes #18908)

---
 youtube_dl/extractor/videomore.py | 96 ++++++++++++++++++++++++++++---
 1 file changed, 88 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 9b56630de..e3eda3327 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -4,8 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    orderedSet,
+    parse_duration,
+    str_or_none,
+    unified_strdate,
+    url_or_none,
     xpath_element,
     xpath_text,
 )
@@ -13,7 +19,19 @@ from ..utils import (
 
 class VideomoreIE(InfoExtractor):
     IE_NAME = 'videomore'
-    _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
+    _VALID_URL = r'''(?x)
+                    videomore:(?P<sid>\d+)$|
+                    https?://(?:player\.)?videomore\.ru/
+                        (?:
+                            (?:
+                                embed|
+                                [^/]+/[^/]+
+                            )/|
+                            [^/]*\?.*?\btrack_id=
+                        )
+                        (?P<id>\d+)
+                        (?:[/?#&]|\.(?:xml|json)|$)
+                    '''
     _TESTS = [{
         'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',
         'md5': '44455a346edc0d509ac5b5a5b531dc35',
@@ -79,6 +97,9 @@ class VideomoreIE(InfoExtractor):
     }, {
         'url': 'videomore:367617',
         'only_matching': True,
+    }, {
+        'url': 'https://player.videomore.ru/?partner_id=97&track_id=736234&autoplay=0&userToken=',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -136,7 +157,7 @@ class VideomoreIE(InfoExtractor):
 
 class VideomoreVideoIE(InfoExtractor):
     IE_NAME = 'videomore:video'
-    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)[/?#&]*$'
+    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)(?:/*|[?#&].*?)$'
     _TESTS = [{
         # single video with og:video:iframe
         'url': 'http://videomore.ru/elki_3',
@@ -176,6 +197,9 @@ class VideomoreVideoIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://videomore.ru/molodezhka/6_sezon/29_seriya?utm_so',
+        'only_matching': True,
     }]
 
     @classmethod
@@ -196,13 +220,16 @@ class VideomoreVideoIE(InfoExtractor):
                  r'track-id=["\'](\d+)',
                  r'xcnt_product_id\s*=\s*(\d+)'), webpage, 'video id')
             video_url = 'videomore:%s' % video_id
+        else:
+            video_id = None
 
-        return self.url_result(video_url, VideomoreIE.ie_key())
+        return self.url_result(
+            video_url, ie=VideomoreIE.ie_key(), video_id=video_id)
 
 
 class VideomoreSeasonIE(InfoExtractor):
     IE_NAME = 'videomore:season'
-    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)[/?#&]*$'
+    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)(?:/*|[?#&].*?)$'
     _TESTS = [{
         'url': 'http://videomore.ru/molodezhka/sezon_promo',
         'info_dict': {
@@ -210,8 +237,16 @@ class VideomoreSeasonIE(InfoExtractor):
             'title': 'Молодежка Промо',
         },
         'playlist_mincount': 12,
+    }, {
+        'url': 'http://videomore.ru/molodezhka/sezon_promo?utm_so',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return (False if (VideomoreIE.suitable(url) or VideomoreVideoIE.suitable(url))
+                else super(VideomoreSeasonIE, cls).suitable(url))
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
@@ -219,9 +254,54 @@ class VideomoreSeasonIE(InfoExtractor):
 
         title = self._og_search_title(webpage)
 
-        entries = [
-            self.url_result(item) for item in re.findall(
-                r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
-                % display_id, webpage)]
+        data = self._parse_json(
+            self._html_search_regex(
+                r'\bclass=["\']seasons-tracks["\'][^>]+\bdata-custom-data=(["\'])(?P<value>{.+?})\1',
+                webpage, 'data', default='{}', group='value'),
+            display_id, fatal=False)
+
+        entries = []
+
+        if data:
+            episodes = data.get('episodes')
+            if isinstance(episodes, list):
+                for ep in episodes:
+                    if not isinstance(ep, dict):
+                        continue
+                    ep_id = int_or_none(ep.get('id'))
+                    ep_url = url_or_none(ep.get('url'))
+                    if ep_id:
+                        e = {
+                            'url': 'videomore:%s' % ep_id,
+                            'id': compat_str(ep_id),
+                        }
+                    elif ep_url:
+                        e = {'url': ep_url}
+                    else:
+                        continue
+                    e.update({
+                        '_type': 'url',
+                        'ie_key': VideomoreIE.ie_key(),
+                        'title': str_or_none(ep.get('title')),
+                        'thumbnail': url_or_none(ep.get('image')),
+                        'duration': parse_duration(ep.get('duration')),
+                        'episode_number': int_or_none(ep.get('number')),
+                        'upload_date': unified_strdate(ep.get('date')),
+                    })
+                    entries.append(e)
+
+        if not entries:
+            entries = [
+                self.url_result(
+                    'videomore:%s' % video_id, ie=VideomoreIE.ie_key(),
+                    video_id=video_id)
+                for video_id in orderedSet(re.findall(
+                    r':(?:id|key)=["\'](\d+)["\']', webpage))]
+
+        if not entries:
+            entries = [
+                self.url_result(item) for item in re.findall(
+                    r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
+                    % display_id, webpage)]
 
         return self.playlist_result(entries, display_id, title)

From 4b85f0f9db9329ef1774a66c3e2fd4da558a5201 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 22 Jan 2019 14:38:29 +0100
Subject: [PATCH 345/510] [vrv] add support for authentication(closes #14307)

---
 youtube_dl/extractor/vrv.py | 95 ++++++++++++++++++++++---------------
 1 file changed, 56 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 483a3be3a..014513051 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -11,10 +11,12 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_urllib_parse_urlencode,
     compat_urllib_parse,
 )
 from ..utils import (
+    ExtractorError,
     float_or_none,
     int_or_none,
 )
@@ -24,29 +26,41 @@ class VRVBaseIE(InfoExtractor):
     _API_DOMAIN = None
     _API_PARAMS = {}
     _CMS_SIGNING = {}
+    _TOKEN = None
+    _TOKEN_SECRET = ''
 
     def _call_api(self, path, video_id, note, data=None):
+        # https://tools.ietf.org/html/rfc5849#section-3
         base_url = self._API_DOMAIN + '/core/' + path
-        encoded_query = compat_urllib_parse_urlencode({
+        query = {
             'oauth_consumer_key': self._API_PARAMS['oAuthKey'],
             'oauth_nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
             'oauth_signature_method': 'HMAC-SHA1',
             'oauth_timestamp': int(time.time()),
-            'oauth_version': '1.0',
-        })
+        }
+        if self._TOKEN:
+            query['oauth_token'] = self._TOKEN
+        encoded_query = compat_urllib_parse_urlencode(query)
         headers = self.geo_verification_headers()
         if data:
             data = json.dumps(data).encode()
             headers['Content-Type'] = 'application/json'
-        method = 'POST' if data else 'GET'
-        base_string = '&'.join([method, compat_urllib_parse.quote(base_url, ''), compat_urllib_parse.quote(encoded_query, '')])
+        base_string = '&'.join([
+            'POST' if data else 'GET',
+            compat_urllib_parse.quote(base_url, ''),
+            compat_urllib_parse.quote(encoded_query, '')])
         oauth_signature = base64.b64encode(hmac.new(
-            (self._API_PARAMS['oAuthSecret'] + '&').encode('ascii'),
+            (self._API_PARAMS['oAuthSecret'] + '&' + self._TOKEN_SECRET).encode('ascii'),
             base_string.encode(), hashlib.sha1).digest()).decode()
         encoded_query += '&oauth_signature=' + compat_urllib_parse.quote(oauth_signature, '')
-        return self._download_json(
-            '?'.join([base_url, encoded_query]), video_id,
-            note='Downloading %s JSON metadata' % note, headers=headers, data=data)
+        try:
+            return self._download_json(
+                '?'.join([base_url, encoded_query]), video_id,
+                note='Downloading %s JSON metadata' % note, headers=headers, data=data)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                raise ExtractorError(json.loads(e.cause.read().decode())['message'], expected=True)
+            raise
 
     def _call_cms(self, path, video_id, note):
         if not self._CMS_SIGNING:
@@ -55,19 +69,22 @@ class VRVBaseIE(InfoExtractor):
             self._API_DOMAIN + path, video_id, query=self._CMS_SIGNING,
             note='Downloading %s JSON metadata' % note, headers=self.geo_verification_headers())
 
-    def _set_api_params(self, webpage, video_id):
-        if not self._API_PARAMS:
-            self._API_PARAMS = self._parse_json(self._search_regex(
-                r'window\.__APP_CONFIG__\s*=\s*({.+?})</script>',
-                webpage, 'api config'), video_id)['cxApiParams']
-            self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
-
     def _get_cms_resource(self, resource_key, video_id):
         return self._call_api(
             'cms_resource', video_id, 'resource path', data={
                 'resource_key': resource_key,
             })['__links__']['cms_resource']['href']
 
+    def _real_initialize(self):
+        webpage = self._download_webpage(
+            'https://vrv.co/', None, headers=self.geo_verification_headers())
+        self._API_PARAMS = self._parse_json(self._search_regex(
+            [
+                r'window\.__APP_CONFIG__\s*=\s*({.+?})(?:</script>|;)',
+                r'window\.__APP_CONFIG__\s*=\s*({.+})'
+            ], webpage, 'app config'), None)['cxApiParams']
+        self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
+
 
 class VRVIE(VRVBaseIE):
     IE_NAME = 'vrv'
@@ -86,6 +103,22 @@ class VRVIE(VRVBaseIE):
             'skip_download': True,
         },
     }]
+    _NETRC_MACHINE = 'vrv'
+
+    def _real_initialize(self):
+        super(VRVIE, self)._real_initialize()
+
+        email, password = self._get_login_info()
+        if email is None:
+            return
+
+        token_credentials = self._call_api(
+            'authenticate/by:credentials', None, 'Token Credentials', data={
+                'email': email,
+                'password': password,
+            })
+        self._TOKEN = token_credentials['oauth_token']
+        self._TOKEN_SECRET = token_credentials['oauth_token_secret']
 
     def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang):
         if not url or stream_format not in ('hls', 'dash'):
@@ -116,28 +149,16 @@ class VRVIE(VRVBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, video_id,
-            headers=self.geo_verification_headers())
-        media_resource = self._parse_json(self._search_regex(
-            [
-                r'window\.__INITIAL_STATE__\s*=\s*({.+?})(?:</script>|;)',
-                r'window\.__INITIAL_STATE__\s*=\s*({.+})'
-            ], webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
 
-        video_data = media_resource.get('json')
-        if not video_data:
-            self._set_api_params(webpage, video_id)
-            episode_path = self._get_cms_resource(
-                'cms:/episodes/' + video_id, video_id)
-            video_data = self._call_cms(episode_path, video_id, 'video')
+        episode_path = self._get_cms_resource(
+            'cms:/episodes/' + video_id, video_id)
+        video_data = self._call_cms(episode_path, video_id, 'video')
         title = video_data['title']
 
-        streams_json = media_resource.get('streams', {}).get('json', {})
-        if not streams_json:
-            self._set_api_params(webpage, video_id)
-            streams_path = video_data['__links__']['streams']['href']
-            streams_json = self._call_cms(streams_path, video_id, 'streams')
+        streams_path = video_data['__links__'].get('streams', {}).get('href')
+        if not streams_path:
+            self.raise_login_required()
+        streams_json = self._call_cms(streams_path, video_id, 'streams')
 
         audio_locale = streams_json.get('audio_locale')
         formats = []
@@ -202,11 +223,7 @@ class VRVSeriesIE(VRVBaseIE):
 
     def _real_extract(self, url):
         series_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, series_id,
-            headers=self.geo_verification_headers())
 
-        self._set_api_params(webpage, series_id)
         seasons_path = self._get_cms_resource(
             'cms:/seasons?series_id=' + series_id, series_id)
         seasons_data = self._call_cms(seasons_path, series_id, 'seasons')

From 503b604a316837b9dd6ef32045e4e9bbfb6a1363 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 22 Jan 2019 18:21:37 +0100
Subject: [PATCH 346/510] [vrv] fix oauth signing for python 2(#14307)

---
 youtube_dl/extractor/vrv.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 014513051..6c060ae76 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -32,14 +32,14 @@ class VRVBaseIE(InfoExtractor):
     def _call_api(self, path, video_id, note, data=None):
         # https://tools.ietf.org/html/rfc5849#section-3
         base_url = self._API_DOMAIN + '/core/' + path
-        query = {
-            'oauth_consumer_key': self._API_PARAMS['oAuthKey'],
-            'oauth_nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
-            'oauth_signature_method': 'HMAC-SHA1',
-            'oauth_timestamp': int(time.time()),
-        }
+        query = [
+            ('oauth_consumer_key', self._API_PARAMS['oAuthKey']),
+            ('oauth_nonce', ''.join([random.choice(string.ascii_letters) for _ in range(32)])),
+            ('oauth_signature_method', 'HMAC-SHA1'),
+            ('oauth_timestamp', int(time.time())),
+        ]
         if self._TOKEN:
-            query['oauth_token'] = self._TOKEN
+            query.append(('oauth_token', self._TOKEN))
         encoded_query = compat_urllib_parse_urlencode(query)
         headers = self.geo_verification_headers()
         if data:

From 278d061a0c5eae20963c0a6df4b9b13fd1537186 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Jan 2019 03:51:29 +0700
Subject: [PATCH 347/510] [pornhub] Bypass scrape detection (closes #5930)

---
 youtube_dl/extractor/pornhub.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index e377de196..f5f3e6593 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -10,7 +10,9 @@ from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
     compat_str,
+    compat_urllib_request,
 )
+from .openload import PhantomJSwrapper
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -126,6 +128,26 @@ class PornHubIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _download_webpage_handle(self, *args, **kwargs):
+        def dl(*args, **kwargs):
+            return super(PornHubIE, self)._download_webpage_handle(*args, **kwargs)
+
+        webpage, urlh = dl(*args, **kwargs)
+
+        if any(re.search(p, webpage) for p in (
+                r'<body\b[^>]+\bonload=["\']go\(\)',
+                r'document\.cookie\s*=\s*["\']RNKEY=',
+                r'document\.location\.reload\(true\)')):
+            url_or_request = args[0]
+            url = (url_or_request.get_full_url()
+                   if isinstance(url_or_request, compat_urllib_request.Request)
+                   else url_or_request)
+            phantom = PhantomJSwrapper(self, required_version='2.0')
+            phantom.get(url, html=webpage)
+            webpage, urlh = dl(*args, **kwargs)
+
+        return webpage, urlh
+
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(

From 6510a3aa971c00525969040ad654249c0c73f125 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Jan 2019 03:55:41 +0700
Subject: [PATCH 348/510] [crunchyroll] Extend _VALID_URL (closes #18955)

---
 youtube_dl/extractor/crunchyroll.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4a68d092b..5e2cbe41d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -144,7 +144,7 @@ class CrunchyrollBaseIE(InfoExtractor):
 
 class CrunchyrollIE(CrunchyrollBaseIE, VRVIE):
     IE_NAME = 'crunchyroll'
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|(?:[^/]*/){1,2}[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
@@ -269,6 +269,9 @@ class CrunchyrollIE(CrunchyrollBaseIE, VRVIE):
     }, {
         'url': 'http://www.crunchyroll.com/media-723735',
         'only_matching': True,
+    }, {
+        'url': 'https://www.crunchyroll.com/en-gb/mob-psycho-100/episode-2-urban-legends-encountering-rumors-780921',
+        'only_matching': True,
     }]
 
     _FORMAT_IDS = {

From 71a1f61700789fb0d61fc6ad9681b6f0899d2f51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Jan 2019 04:12:06 +0700
Subject: [PATCH 349/510] [pornhub] Apply scrape detection bypass for all
 extractors

---
 youtube_dl/extractor/pornhub.py | 46 +++++++++++++++++----------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index f5f3e6593..be93d5d48 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -24,7 +24,29 @@ from ..utils import (
 )
 
 
-class PornHubIE(InfoExtractor):
+class PornHubBaseIE(InfoExtractor):
+    def _download_webpage_handle(self, *args, **kwargs):
+        def dl(*args, **kwargs):
+            return super(PornHubBaseIE, self)._download_webpage_handle(*args, **kwargs)
+
+        webpage, urlh = dl(*args, **kwargs)
+
+        if any(re.search(p, webpage) for p in (
+                r'<body\b[^>]+\bonload=["\']go\(\)',
+                r'document\.cookie\s*=\s*["\']RNKEY=',
+                r'document\.location\.reload\(true\)')):
+            url_or_request = args[0]
+            url = (url_or_request.get_full_url()
+                   if isinstance(url_or_request, compat_urllib_request.Request)
+                   else url_or_request)
+            phantom = PhantomJSwrapper(self, required_version='2.0')
+            phantom.get(url, html=webpage)
+            webpage, urlh = dl(*args, **kwargs)
+
+        return webpage, urlh
+
+
+class PornHubIE(PornHubBaseIE):
     IE_DESC = 'PornHub and Thumbzilla'
     _VALID_URL = r'''(?x)
                     https?://
@@ -128,26 +150,6 @@ class PornHubIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _download_webpage_handle(self, *args, **kwargs):
-        def dl(*args, **kwargs):
-            return super(PornHubIE, self)._download_webpage_handle(*args, **kwargs)
-
-        webpage, urlh = dl(*args, **kwargs)
-
-        if any(re.search(p, webpage) for p in (
-                r'<body\b[^>]+\bonload=["\']go\(\)',
-                r'document\.cookie\s*=\s*["\']RNKEY=',
-                r'document\.location\.reload\(true\)')):
-            url_or_request = args[0]
-            url = (url_or_request.get_full_url()
-                   if isinstance(url_or_request, compat_urllib_request.Request)
-                   else url_or_request)
-            phantom = PhantomJSwrapper(self, required_version='2.0')
-            phantom.get(url, html=webpage)
-            webpage, urlh = dl(*args, **kwargs)
-
-        return webpage, urlh
-
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
@@ -329,7 +331,7 @@ class PornHubIE(InfoExtractor):
         }
 
 
-class PornHubPlaylistBaseIE(InfoExtractor):
+class PornHubPlaylistBaseIE(PornHubBaseIE):
     def _extract_entries(self, webpage, host):
         # Only process container div with main playlist content skipping
         # drop-down menu that uses similar pattern for videos (see

From 0670bdd8f2bca39fdeadc63e1cd53b5d5b3e638c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Jan 2019 04:43:55 +0700
Subject: [PATCH 350/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 902301765..687796a0e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,32 @@
+version <unreleased>
+
+Core
+* [utils] Fix urljoin for paths with non-http(s) schemes
+* [extractor/common] Improve jwplayer relative URL handling (#18892)
++ [YoutubeDL] Add negation support for string comparisons in format selection
+  expressions (#18600, #18805)
+* [extractor/common] Improve HLS video-only format detection (#18923)
+
+Extractors
+* [crunchyroll] Extend URL regular expression (#18955)
+* [pornhub] Bypass scrape detection (#4822, #5930, #7074, #10175, #12722,
+  #17197, #18338 #18842, #18899)
++ [vrv] Add support for authentication (#14307)
+* [videomore:season] Fix extraction
+* [videomore] Improve extraction (#18908)
++ [tnaflix] Pass Referer in metadata request (#18925)
+* [radiocanada] Relax DRM check (#18608, #18609)
+* [vimeo] Fix video password verification for videos protected by
+  Referer HTTP header
++ [hketv] Add support for hkedcity.net (#18696)
++ [streamango] Add support for fruithosts.net (#18710)
++ [instagram] Add support for tags (#18757)
++ [odnoklassniki] Detect paid videos (#18876)
+* [ted] Correct acodec for HTTP formats (#18923)
+* [cartoonnetwork] Fix extraction (#15664, #17224)
+* [vimeo] Fix extraction for password protected player URLs (#18889)
+
+
 version 2019.01.17
 
 Extractors

From 435e382423f860aca82a58d7c3db58cbfa242b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Jan 2019 04:46:55 +0700
Subject: [PATCH 351/510] release 2019.01.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 841bca914..db3ebaeed 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.17*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.17
+[debug] youtube-dl version 2019.01.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 687796a0e..55ad44315 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.23
 
 Core
 * [utils] Fix urljoin for paths with non-http(s) schemes
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c01409419..d759d0273 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -361,6 +361,7 @@
  - **hitbox**
  - **hitbox:live**
  - **HitRecord**
+ - **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
  - **HornBunny**
  - **HotNewHipHop**
  - **hotstar**
@@ -386,6 +387,7 @@
  - **IndavideoEmbed**
  - **InfoQ**
  - **Instagram**
+ - **instagram:tag**: Instagram hashtag search
  - **instagram:user**: Instagram user profile
  - **Internazionale**
  - **InternetVideoArchive**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ea3f62928..d77949eed 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.17'
+__version__ = '2019.01.23'

From e118a8794ffe5a3a414afd489726f34d753b0b23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Jan 2019 01:34:41 +0700
Subject: [PATCH 352/510] [YoutubeDL] Fix typo in string negation
 implementation and add more tests (closes #18961)

---
 test/test_YoutubeDL.py  | 30 +++++++++++++++++++++++++++---
 youtube_dl/YoutubeDL.py |  2 +-
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index df8994b84..1d7452744 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -242,6 +242,7 @@ class TestFormatSelection(unittest.TestCase):
     def test_format_selection_string_ops(self):
         formats = [
             {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
+            {'format_id': 'zxc-cxz', 'ext': 'webm', 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -253,6 +254,11 @@ class TestFormatSelection(unittest.TestCase):
 
         # does not equal (!=)
         ydl = YDL({'format': '[format_id!=abc-cba]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'zxc-cxz')
+
+        ydl = YDL({'format': '[format_id!=abc-cba][format_id!=zxc-cxz]'})
         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
 
         # starts with (^=)
@@ -262,7 +268,12 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], 'abc-cba')
 
         # does not start with (!^=)
-        ydl = YDL({'format': '[format_id!^=abc-cba]'})
+        ydl = YDL({'format': '[format_id!^=abc]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'zxc-cxz')
+
+        ydl = YDL({'format': '[format_id!^=abc][format_id!^=zxc]'})
         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
 
         # ends with ($=)
@@ -272,16 +283,29 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], 'abc-cba')
 
         # does not end with (!$=)
-        ydl = YDL({'format': '[format_id!$=abc-cba]'})
+        ydl = YDL({'format': '[format_id!$=cba]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'zxc-cxz')
+
+        ydl = YDL({'format': '[format_id!$=cba][format_id!$=cxz]'})
         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
 
         # contains (*=)
-        ydl = YDL({'format': '[format_id*=-]'})
+        ydl = YDL({'format': '[format_id*=bc-cb]'})
         ydl.process_ie_result(info_dict.copy())
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'abc-cba')
 
         # does not contain (!*=)
+        ydl = YDL({'format': '[format_id!*=bc-cb]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'zxc-cxz')
+
+        ydl = YDL({'format': '[format_id!*=abc][format_id!*=zxc]'})
+        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
+
         ydl = YDL({'format': '[format_id!*=-]'})
         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a827414dc..80ed8d7e5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1078,7 +1078,7 @@ class YoutubeDL(object):
                 comparison_value = m.group('value')
                 str_op = STR_OPERATORS[m.group('op')]
                 if m.group('negation'):
-                    op = lambda attr, value: not str_op
+                    op = lambda attr, value: not str_op(attr, value)
                 else:
                     op = str_op
 

From 7d311586eda9eae9430da3f6d18932d79127daa7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Jan 2019 01:44:09 +0700
Subject: [PATCH 353/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 55ad44315..a28c6e951 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Fix negation for string operators in format selection (#18961)
+
+
 version 2019.01.23
 
 Core

From a1e171233d86a064865353cc820969c10cb1f251 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Jan 2019 01:46:23 +0700
Subject: [PATCH 354/510] release 2019.01.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index db3ebaeed..63aefe013 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.24*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.23
+[debug] youtube-dl version 2019.01.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a28c6e951..1fda747bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.24
 
 Core
 * [YoutubeDL] Fix negation for string operators in format selection (#18961)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d77949eed..18c1f8d4c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.23'
+__version__ = '2019.01.24'

From 9713d1d1e0a7eff5c1b9873a2f4f054111a568ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Jan 2019 02:30:12 +0700
Subject: [PATCH 355/510] [openload] Add support for oload.club (closes #18969)

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index cf51e4770..b713e78b8 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -249,7 +249,7 @@ class OpenloadIE(InfoExtractor):
                             (?:www\.)?
                             (?:
                                 openload\.(?:co|io|link)|
-                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun)
+                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club)
                             )
                         )/
                         (?:f|embed)/
@@ -334,6 +334,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.fun/f/gb6G1H4sHXY',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.club/f/Nr1L-aZ2dbQ',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From 118afcf52ff23726e5f0c436083710f5c63230fa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 23 Jan 2019 22:16:21 +0100
Subject: [PATCH 356/510] [go] fix adobe pass requests for Disney Now(closes
 #18901)

---
 youtube_dl/extractor/go.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index ec9dd6e3a..206d89e82 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -25,15 +25,15 @@ class GoIE(AdobePassIE):
         },
         'watchdisneychannel': {
             'brand': '004',
-            'requestor_id': 'Disney',
+            'resource_id': 'Disney',
         },
         'watchdisneyjunior': {
             'brand': '008',
-            'requestor_id': 'DisneyJunior',
+            'resource_id': 'DisneyJunior',
         },
         'watchdisneyxd': {
             'brand': '009',
-            'requestor_id': 'DisneyXD',
+            'resource_id': 'DisneyXD',
         }
     }
     _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:(?:[^/]+/)*(?P<id>vdka\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))'\
@@ -130,8 +130,8 @@ class GoIE(AdobePassIE):
                     'device': '001',
                 }
                 if video_data.get('accesslevel') == '1':
-                    requestor_id = site_info['requestor_id']
-                    resource = self._get_mvpd_resource(
+                    requestor_id = site_info.get('requestor_id', 'DisneyChannels')
+                    resource = site_info.get('resource_id') or self._get_mvpd_resource(
                         requestor_id, title, video_id, None)
                     auth = self._extract_mvpd_auth(
                         url, video_id, requestor_id, resource)

From eb35b163adf61f8ff0ee6c504e98bc94db16e705 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 24 Jan 2019 20:23:04 +0100
Subject: [PATCH 357/510] [postprocessor/ffmpeg] fallback to ffmpeg/avconv for
 audio codec detection(closes #681)

---
 youtube_dl/postprocessor/ffmpeg.py | 53 +++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 39a905380..b952b0970 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -9,9 +9,6 @@ import re
 
 from .common import AudioConversionError, PostProcessor
 
-from ..compat import (
-    compat_subprocess_get_DEVNULL,
-)
 from ..utils import (
     encodeArgument,
     encodeFilename,
@@ -165,27 +162,45 @@ class FFmpegPostProcessor(PostProcessor):
         return self._paths[self.probe_basename]
 
     def get_audio_codec(self, path):
-        if not self.probe_available:
-            raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
+        if not self.probe_available and not self.available:
+            raise PostProcessingError('ffprobe/avprobe and ffmpeg/avconv not found. Please install one.')
         try:
-            cmd = [
-                encodeFilename(self.probe_executable, True),
-                encodeArgument('-show_streams'),
-                encodeFilename(self._ffmpeg_filename_argument(path), True)]
+            if self.probe_available:
+                cmd = [
+                    encodeFilename(self.probe_executable, True),
+                    encodeArgument('-show_streams')]
+            else:
+                cmd = [
+                    encodeFilename(self.executable, True),
+                    encodeArgument('-i')]
+            cmd.append(encodeFilename(self._ffmpeg_filename_argument(path), True))
             if self._downloader.params.get('verbose', False):
-                self._downloader.to_screen('[debug] %s command line: %s' % (self.basename, shell_quote(cmd)))
-            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            output = handle.communicate()[0]
-            if handle.wait() != 0:
+                self._downloader.to_screen(
+                    '[debug] %s command line: %s' % (self.basename, shell_quote(cmd)))
+            handle = subprocess.Popen(
+                cmd, stderr=subprocess.PIPE,
+                stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            stdout_data, stderr_data = handle.communicate()
+            expected_ret = 0 if self.probe_available else 1
+            if handle.wait() != expected_ret:
                 return None
         except (IOError, OSError):
             return None
-        audio_codec = None
-        for line in output.decode('ascii', 'ignore').split('\n'):
-            if line.startswith('codec_name='):
-                audio_codec = line.split('=')[1].strip()
-            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
-                return audio_codec
+        output = (stdout_data if self.probe_available else stderr_data).decode('ascii', 'ignore')
+        if self.probe_available:
+            audio_codec = None
+            for line in output.split('\n'):
+                if line.startswith('codec_name='):
+                    audio_codec = line.split('=')[1].strip()
+                elif line.strip() == 'codec_type=audio' and audio_codec is not None:
+                    return audio_codec
+        else:
+            # Stream #FILE_INDEX:STREAM_INDEX[STREAM_ID](LANGUAGE): CODEC_TYPE: CODEC_NAME
+            mobj = re.search(
+                r'Stream\s*#\d+:\d+(?:\[0x[0-9a-f]+\])?(?:\([a-z]{3}\))?:\s*Audio:\s*([0-9a-z]+)',
+                output)
+            if mobj:
+                return mobj.group(1)
         return None
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):

From 0eba178fce80923515f4a9ac411e46648a19d78c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Jan 2019 04:04:58 +0700
Subject: [PATCH 358/510] [nhk] Extend _VALID_URL (closes #18968)

---
 youtube_dl/extractor/nhk.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index 5c8cd76dc..d4acbcc3e 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -5,8 +5,8 @@ from ..utils import ExtractorError
 
 
 class NhkVodIE(InfoExtractor):
-    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/vod/(?P<id>[^/]+/[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/(?:vod|ondemand)/(?P<id>[^/]+/[^/?#&]+)'
+    _TESTS = [{
         # Videos available only for a limited period of time. Visit
         # http://www3.nhk.or.jp/nhkworld/en/vod/ for working samples.
         'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815',
@@ -19,7 +19,10 @@ class NhkVodIE(InfoExtractor):
             'episode': 'The Kimono as Global Fashion',
         },
         'skip': 'Videos available only for a limited period of time',
-    }
+    }, {
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
+        'only_matching': True,
+    }]
     _API_URL = 'http://api.nhk.or.jp/nhkworld/vodesdlist/v1/all/all/all.json?apikey=EJfK8jdS57GqlupFgAfAAwr573q01y6k'
 
     def _real_extract(self, url):

From 1602a240a7742f9e0a02b3f0effd215d00d859f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Jan 2019 04:16:49 +0700
Subject: [PATCH 359/510] [drtv] Fix extraction (closes #18989)

---
 youtube_dl/extractor/drtv.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index f757745ba..8d63ca433 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -77,10 +77,9 @@ class DRTVIE(InfoExtractor):
                 r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
             webpage, 'video id')
 
-        programcard = self._download_json(
-            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
-            video_id, 'Downloading video JSON')
-        data = programcard['Data'][0]
+        data = self._download_json(
+            'https://www.dr.dk/mu-online/api/1.4/programcard/%s' % video_id,
+            video_id, 'Downloading video JSON', query={'expanded': 'true'})
 
         title = remove_end(self._og_search_title(
             webpage, default=None), ' | TV | DR') or data['Title']
@@ -97,7 +96,7 @@ class DRTVIE(InfoExtractor):
         formats = []
         subtitles = {}
 
-        for asset in data['Assets']:
+        for asset in [data['PrimaryAsset']]:
             kind = asset.get('Kind')
             if kind == 'Image':
                 thumbnail = asset.get('Uri')

From ae18d58297c16a300a89693360efd19be4a97e92 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 25 Jan 2019 11:01:27 +0100
Subject: [PATCH 360/510] [usatoday] fix extraction for videos with custom
 brightcove partner id(closes #18990)

---
 youtube_dl/extractor/usatoday.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/usatoday.py b/youtube_dl/extractor/usatoday.py
index e5678dc78..b2103448d 100644
--- a/youtube_dl/extractor/usatoday.py
+++ b/youtube_dl/extractor/usatoday.py
@@ -3,21 +3,23 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     get_element_by_attribute,
     parse_duration,
+    try_get,
     update_url_query,
-    ExtractorError,
 )
 from ..compat import compat_str
 
 
 class USATodayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?usatoday\.com/(?:[^/]+/)*(?P<id>[^?/#]+)'
-    _TEST = {
+    _TESTS = [{
+        # Brightcove Partner ID = 29906170001
         'url': 'http://www.usatoday.com/media/cinematic/video/81729424/us-france-warn-syrian-regime-ahead-of-new-peace-talks/',
-        'md5': '4d40974481fa3475f8bccfd20c5361f8',
+        'md5': '033587d2529dc3411a1ab3644c3b8827',
         'info_dict': {
-            'id': '81729424',
+            'id': '4799374959001',
             'ext': 'mp4',
             'title': 'US, France warn Syrian regime ahead of new peace talks',
             'timestamp': 1457891045,
@@ -25,8 +27,20 @@ class USATodayIE(InfoExtractor):
             'uploader_id': '29906170001',
             'upload_date': '20160313',
         }
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/29906170001/38a9eecc-bdd8-42a3-ba14-95397e48b3f8_default/index.html?videoId=%s'
+    }, {
+        # ui-video-data[asset_metadata][items][brightcoveaccount] = 28911775001
+        'url': 'https://www.usatoday.com/story/tech/science/2018/08/21/yellowstone-supervolcano-eruption-stop-worrying-its-blow/973633002/',
+        'info_dict': {
+            'id': '5824495846001',
+            'ext': 'mp4',
+            'title': 'Yellowstone more likely to crack rather than explode',
+            'timestamp': 1534790612,
+            'description': 'md5:3715e7927639a4f16b474e9391687c62',
+            'uploader_id': '28911775001',
+            'upload_date': '20180820',
+        }
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -35,10 +49,11 @@ class USATodayIE(InfoExtractor):
         if not ui_video_data:
             raise ExtractorError('no video on the webpage', expected=True)
         video_data = self._parse_json(ui_video_data, display_id)
+        item = try_get(video_data, lambda x: x['asset_metadata']['items'], dict) or {}
 
         return {
             '_type': 'url_transparent',
-            'url': self.BRIGHTCOVE_URL_TEMPLATE % video_data['brightcove_id'],
+            'url': self.BRIGHTCOVE_URL_TEMPLATE % (item.get('brightcoveaccount', '29906170001'), item.get('brightcoveid') or video_data['brightcove_id']),
             'id': compat_str(video_data['id']),
             'title': video_data['title'],
             'thumbnail': video_data.get('thumbnail'),

From 252abb1e8b881aa9d3942c436711ac33235b37cd Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 26 Jan 2019 15:29:19 +0700
Subject: [PATCH 361/510] [README.md] Mention more convenience extraction
 functions

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4ba982907..c1572f771 100644
--- a/README.md
+++ b/README.md
@@ -1213,7 +1213,7 @@ Incorrect:
 'PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
 ```
 
-### Use safe conversion functions
+### Use convenience conversion and parsing functions
 
 Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
 
@@ -1221,6 +1221,8 @@ Use `url_or_none` for safe URL processing.
 
 Use `try_get` for safe metadata extraction from parsed JSON.
 
+Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. 
+
 Explore [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
 
 #### More examples

From 845333acf6280761d19f91b3e018c418d922a0de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Jan 2019 04:14:54 +0700
Subject: [PATCH 362/510] [wakanim] Add extractor (closes #14374)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/wakanim.py    | 55 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/wakanim.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 574a47e6d..2ffcffa9e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1373,6 +1373,7 @@ from .vuclip import VuClipIE
 from .vvvvid import VVVVIDIE
 from .vyborymos import VyboryMosIE
 from .vzaar import VzaarIE
+from .wakanim import WakanimIE
 from .walla import WallaIE
 from .washingtonpost import (
     WashingtonPostIE,
diff --git a/youtube_dl/extractor/wakanim.py b/youtube_dl/extractor/wakanim.py
new file mode 100644
index 000000000..1d588bdd6
--- /dev/null
+++ b/youtube_dl/extractor/wakanim.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    merge_dicts,
+    urljoin,
+)
+
+
+class WakanimIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?wakanim\.tv/[^/]+/v2/catalogue/episode/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.wakanim.tv/de/v2/catalogue/episode/2997/the-asterisk-war-omu-staffel-1-episode-02-omu',
+        'info_dict': {
+            'id': '2997',
+            'ext': 'mp4',
+            'title': 'Episode 02',
+            'description': 'md5:2927701ea2f7e901de8bfa8d39b2852d',
+            'series': 'The Asterisk War  (OmU.)',
+            'season_number': 1,
+            'episode': 'Episode 02',
+            'episode_number': 2,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        m3u8_url = urljoin(url, self._search_regex(
+            r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage, 'm3u8 url',
+            group='url'))
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
+        info = self._search_json_ld(webpage, video_id, default={})
+
+        title = self._search_regex(
+            (r'<h1[^>]+\bclass=["\']episode_h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<span[^>]+\bclass=["\']episode_title["\'][^>]*>(?P<title>[^<]+)'),
+            webpage, 'title', default=None, group='title')
+
+        return merge_dicts(info, {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        })

From 458fd30f56785d514862dcb8a604a329d8e29ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Jan 2019 04:36:58 +0700
Subject: [PATCH 363/510] [extractor/common] Extract season in _json_ld

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 95456b291..c4ea2882f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1249,7 +1249,10 @@ class InfoExtractor(object):
                         info['title'] = episode_name
                     part_of_season = e.get('partOfSeason')
                     if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
-                        info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
+                        info.update({
+                            'season': unescapeHTML(part_of_season.get('name')),
+                            'season_number': int_or_none(part_of_season.get('seasonNumber')),
+                        })
                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
                     if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
                         info['series'] = unescapeHTML(part_of_series.get('name'))

From 30cd1a5f3920d7485225a5d57b6ce41be4cde672 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 26 Jan 2019 22:52:54 +0100
Subject: [PATCH 364/510] [wakanim] detect DRM protected videos

---
 youtube_dl/extractor/wakanim.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wakanim.py b/youtube_dl/extractor/wakanim.py
index 1d588bdd6..f9a2395d9 100644
--- a/youtube_dl/extractor/wakanim.py
+++ b/youtube_dl/extractor/wakanim.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     merge_dicts,
     urljoin,
 )
@@ -10,7 +11,7 @@ from ..utils import (
 
 class WakanimIE(InfoExtractor):
     _VALID_URL = r'https://(?:www\.)?wakanim\.tv/[^/]+/v2/catalogue/episode/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.wakanim.tv/de/v2/catalogue/episode/2997/the-asterisk-war-omu-staffel-1-episode-02-omu',
         'info_dict': {
             'id': '2997',
@@ -26,7 +27,11 @@ class WakanimIE(InfoExtractor):
             'format': 'bestvideo',
             'skip_download': True,
         },
-    }
+    }, {
+        # DRM Protected
+        'url': 'https://www.wakanim.tv/de/v2/catalogue/episode/7843/sword-art-online-alicization-omu-arc-2-folge-15-omu',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -36,6 +41,12 @@ class WakanimIE(InfoExtractor):
         m3u8_url = urljoin(url, self._search_regex(
             r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage, 'm3u8 url',
             group='url'))
+        # https://docs.microsoft.com/en-us/azure/media-services/previous/media-services-content-protection-overview#streaming-urls
+        encryption = self._search_regex(
+            r'encryption%3D(c(?:enc|bc(?:s-aapl)?))',
+            m3u8_url, 'encryption', default=None)
+        if encryption and encryption in ('cenc', 'cbcs-aapl'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = self._extract_m3u8_formats(
             m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',

From 1fcc91663bc84a599cc613ff1fa0e4bc15f42a9e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Jan 2019 10:53:38 +0100
Subject: [PATCH 365/510] [vice] fix extraction for locked videos(closes
 #16248)

---
 youtube_dl/extractor/vice.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 538258617..8fdfd743d 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -94,7 +94,6 @@ class ViceIE(AdobePassIE):
         'url': 'https://www.viceland.com/en_us/video/thursday-march-1-2018/5a8f2d7ff1cdb332dd446ec1',
         'only_matching': True,
     }]
-    _PREPLAY_HOST = 'vms.vice'
 
     @staticmethod
     def _extract_urls(webpage):
@@ -158,9 +157,8 @@ class ViceIE(AdobePassIE):
         })
 
         try:
-            host = 'www.viceland' if is_locked else self._PREPLAY_HOST
             preplay = self._download_json(
-                'https://%s.com/%s/video/preplay/%s' % (host, locale, video_id),
+                'https://vms.vice.com/%s/video/preplay/%s' % (locale, video_id),
                 video_id, query=query)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401):

From bf8ebc9cfe1ae2b62baf2116f84748db03c4df7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Jan 2019 21:25:43 +0700
Subject: [PATCH 366/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 1fda747bb..9d5c25273 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+version <unreleased>
+
+Core
++ [extractor/common] Extract season in _json_ld
+* [postprocessor/ffmpeg] Fallback to ffmpeg/avconv for audio codec detection
+  (#681)
+
+Extractors
+* [vice] Fix extraction for locked videos (#16248)
++ [wakanim] Detect DRM protected videos
++ [wakanim] Add support for wakanim.tv (#14374)
+* [usatoday] Fix extraction for videos with custom brightcove partner id
+  (#18990)
+* [drtv] Fix extraction (#18989)
+* [nhk] Extend URL regular expression (#18968)
+* [go] Fix Adobe Pass requests for Disney Now (#18901)
++ [openload] Add support for oload.club (#18969)
+
+
 version 2019.01.24
 
 Core

From e71be6ee9f239308765443d49d91358fa306e48a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Jan 2019 21:28:09 +0700
Subject: [PATCH 367/510] release 2019.01.27

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 CONTRIBUTING.md           | 4 +++-
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 63aefe013..f529e3f4b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.24*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.27*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.27**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.24
+[debug] youtube-dl version 2019.01.27
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a71b045d0..6c1739860 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -339,7 +339,7 @@ Incorrect:
 'PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
 ```
 
-### Use safe conversion functions
+### Use convenience conversion and parsing functions
 
 Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
 
@@ -347,6 +347,8 @@ Use `url_or_none` for safe URL processing.
 
 Use `try_get` for safe metadata extraction from parsed JSON.
 
+Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. 
+
 Explore [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
 
 #### More examples
diff --git a/ChangeLog b/ChangeLog
index 9d5c25273..d94fe36ec 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.27
 
 Core
 + [extractor/common] Extract season in _json_ld
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d759d0273..6377bf815 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1069,6 +1069,7 @@
  - **VVVVID**
  - **VyboryMos**
  - **Vzaar**
+ - **Wakanim**
  - **Walla**
  - **WalyTV**
  - **washingtonpost**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 18c1f8d4c..ec89cfc64 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.24'
+__version__ = '2019.01.27'

From 2b3afe6b0fee226bb1e61027d815df088cb40757 Mon Sep 17 00:00:00 2001
From: Andrew Udvare <audvare@gmail.com>
Date: Sun, 27 Jan 2019 22:24:37 -0500
Subject: [PATCH 368/510] [postprocessor/ffmpeg] Disable "Last message
 repeated" messages which cause non-zero exit status

---
 youtube_dl/postprocessor/ffmpeg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index b952b0970..8ef03f43b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -218,6 +218,7 @@ class FFmpegPostProcessor(PostProcessor):
                 encodeFilename(self._ffmpeg_filename_argument(path), True)
             ])
         cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
+               ['-loglevel', 'repeat+info'] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])

From 7f903dd8bfbc9c2de129d5b0be23ef62d8ca3df3 Mon Sep 17 00:00:00 2001
From: Tatsh <Tatsh@users.noreply.github.com>
Date: Mon, 28 Jan 2019 10:57:14 -0500
Subject: [PATCH 369/510] [postprocessor/ffmpeg] Do not copy Apple TV chapter
 tracks while embedding subtitles (closes #19042)

Related issue: https://trac.ffmpeg.org/ticket/6016
---
 youtube_dl/postprocessor/ffmpeg.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index b952b0970..fff2021ff 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -407,6 +407,9 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',
+            # Don't copy Apple TV chapters track, bin_data (see #19042, #19024,
+            # https://trac.ffmpeg.org/ticket/6016)
+            '-map', '-0:d',
         ]
         if information['ext'] == 'mp4':
             opts += ['-c:s', 'mov_text']

From 61ff92e11ea876532697451b1ed727f42274b109 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Jan 2019 01:59:56 +0700
Subject: [PATCH 370/510] [postprocessor/ffmpeg] Wrap loglevel args in
 encodeArgument

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 33dbcad9f..88b9ae9be 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -218,7 +218,7 @@ class FFmpegPostProcessor(PostProcessor):
                 encodeFilename(self._ffmpeg_filename_argument(path), True)
             ])
         cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
-               ['-loglevel', 'repeat+info'] +
+               [encodeArgument('-loglevel'), encodeArgument('repeat+info')] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])

From a81daba2311cb4d6c5bc7e62b47438a78aa5c10f Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Mon, 28 Jan 2019 20:20:46 +0100
Subject: [PATCH 371/510] [zattoo] Add support for tv.salt.ch

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/zattoo.py     | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2ffcffa9e..9d776ff45 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1497,6 +1497,7 @@ from .zattoo import (
     QuantumTVIE,
     QuicklineIE,
     QuicklineLiveIE,
+    SaltTVIE,
     SAKTVIE,
     VTXTVIE,
     WalyTVIE,
diff --git a/youtube_dl/extractor/zattoo.py b/youtube_dl/extractor/zattoo.py
index 896276301..ee514666b 100644
--- a/youtube_dl/extractor/zattoo.py
+++ b/youtube_dl/extractor/zattoo.py
@@ -420,3 +420,14 @@ class EinsUndEinsTVIE(ZattooIE):
         'url': 'https://www.1und1.tv/watch/abc/123-abc',
         'only_matching': True,
     }]
+
+
+class SaltTVIE(ZattooIE):
+    _NETRC_MACHINE = 'salttv'
+    _HOST = 'tv.salt.ch'
+    _VALID_URL = _make_valid_url(ZattooIE._VALID_URL_TEMPLATE, _HOST)
+
+    _TESTS = [{
+        'url': 'https://tv.salt.ch/watch/abc/123-abc',
+        'only_matching': True,
+    }]

From 41c2c254d3c30afde395e8abbe0ced2c53485a78 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 28 Jan 2019 22:39:08 +0100
Subject: [PATCH 372/510] [fox] fix extraction for free videos(#19060)

---
 youtube_dl/extractor/fox.py | 46 ++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index b1c91f095..2d6c97ec9 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -1,10 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-# import json
-# import uuid
+import json
+import uuid
 
 from .adobepass import AdobePassIE
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     parse_age_limit,
@@ -47,38 +48,31 @@ class FOXIE(AdobePassIE):
         'url': 'https://www.nationalgeographic.com/tv/watch/f690e05ebbe23ab79747becd0cc223d1/',
         'only_matching': True,
     }]
-    # _access_token = None
+    _access_token = None
 
-    # def _call_api(self, path, video_id, data=None):
-    #     headers = {
-    #         'X-Api-Key': '238bb0a0c2aba67922c48709ce0c06fd',
-    #     }
-    #     if self._access_token:
-    #         headers['Authorization'] = 'Bearer ' + self._access_token
-    #     return self._download_json(
-    #         'https://api2.fox.com/v2.0/' + path, video_id, data=data, headers=headers)
+    def _call_api(self, path, video_id, data=None):
+        headers = {
+            'X-Api-Key': '238bb0a0c2aba67922c48709ce0c06fd',
+        }
+        if self._access_token:
+            headers['Authorization'] = 'Bearer ' + self._access_token
+        return self._download_json(
+            'https://api2.fox.com/v2.0/' + path,
+            video_id, data=data, headers=headers)
 
-    # def _real_initialize(self):
-    #     self._access_token = self._call_api(
-    #         'login', None, json.dumps({
-    #             'deviceId': compat_str(uuid.uuid4()),
-    #         }).encode())['accessToken']
+    def _real_initialize(self):
+        self._access_token = self._call_api(
+            'login', None, json.dumps({
+                'deviceId': compat_str(uuid.uuid4()),
+            }).encode())['accessToken']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video = self._download_json(
-            'https://api.fox.com/fbc-content/v1_5/video/%s' % video_id,
-            video_id, headers={
-                'apikey': 'abdcbed02c124d393b39e818a4312055',
-                'Content-Type': 'application/json',
-                'Referer': url,
-            })
-        # video = self._call_api('vodplayer/' + video_id, video_id)
+        video = self._call_api('vodplayer/' + video_id, video_id)
 
         title = video['name']
-        release_url = video['videoRelease']['url']
-        # release_url = video['url']
+        release_url = video['url']
 
         data = try_get(
             video, lambda x: x['trackingData']['properties'], dict) or {}

From 6df196f32e68ec22bd854c4d779b9d94e04e63b2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Jan 2019 00:31:49 +0100
Subject: [PATCH 373/510] [fox] add support for locked videos using
 cookies(closes #19060)

---
 youtube_dl/extractor/extractors.py         |  5 ++-
 youtube_dl/extractor/fox.py                | 50 +++++++++++-----------
 youtube_dl/extractor/nationalgeographic.py | 22 ++++++++++
 3 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9d776ff45..b40be42e6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -692,7 +692,10 @@ from .myvi import (
     MyviEmbedIE,
 )
 from .myvidster import MyVidsterIE
-from .nationalgeographic import NationalGeographicVideoIE
+from .nationalgeographic import (
+    NationalGeographicVideoIE,
+    NationalGeographicTVIE,
+)
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 2d6c97ec9..568656542 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -5,19 +5,23 @@ import json
 import uuid
 
 from .adobepass import AdobePassIE
-from ..compat import compat_str
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
+    ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_duration,
     try_get,
     unified_timestamp,
-    update_url_query,
 )
 
 
 class FOXIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?(?:fox\.com|nationalgeographic\.com/tv)/watch/(?P<id>[\da-fA-F]+)'
+    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[\da-fA-F]+)'
     _TESTS = [{
         # clip
         'url': 'https://www.fox.com/watch/4b765a60490325103ea69888fb2bd4e8/',
@@ -32,6 +36,7 @@ class FOXIE(AdobePassIE):
             'upload_date': '20170901',
             'creator': 'FOX',
             'series': 'Gotham',
+            'age_limit': 14,
         },
         'params': {
             'skip_download': True,
@@ -44,15 +49,14 @@ class FOXIE(AdobePassIE):
         # episode, geo-restricted, tv provided required
         'url': 'https://www.fox.com/watch/30056b295fb57f7452aeeb4920bc3024/',
         'only_matching': True,
-    }, {
-        'url': 'https://www.nationalgeographic.com/tv/watch/f690e05ebbe23ab79747becd0cc223d1/',
-        'only_matching': True,
     }]
+    _HOME_PAGE_URL = 'https://www.fox.com/'
+    _API_KEY = 'abdcbed02c124d393b39e818a4312055'
     _access_token = None
 
     def _call_api(self, path, video_id, data=None):
         headers = {
-            'X-Api-Key': '238bb0a0c2aba67922c48709ce0c06fd',
+            'X-Api-Key': self._API_KEY,
         }
         if self._access_token:
             headers['Authorization'] = 'Bearer ' + self._access_token
@@ -61,10 +65,16 @@ class FOXIE(AdobePassIE):
             video_id, data=data, headers=headers)
 
     def _real_initialize(self):
-        self._access_token = self._call_api(
-            'login', None, json.dumps({
-                'deviceId': compat_str(uuid.uuid4()),
-            }).encode())['accessToken']
+        if not self._access_token:
+            mvpd_auth = self._get_cookies(self._HOME_PAGE_URL).get('mvpd-auth')
+            if mvpd_auth:
+                self._access_token = (self._parse_json(compat_urllib_parse_unquote(
+                    mvpd_auth.value), None, fatal=False) or {}).get('accessToken')
+            if not self._access_token:
+                self._access_token = self._call_api(
+                    'login', None, json.dumps({
+                        'deviceId': compat_str(uuid.uuid4()),
+                    }).encode())['accessToken']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -73,25 +83,15 @@ class FOXIE(AdobePassIE):
 
         title = video['name']
         release_url = video['url']
-
-        data = try_get(
-            video, lambda x: x['trackingData']['properties'], dict) or {}
-
-        rating = video.get('contentRating')
-        if data.get('authRequired'):
-            resource = self._get_mvpd_resource(
-                'fbc-fox', title, video.get('guid'), rating)
-            release_url = update_url_query(
-                release_url, {
-                    'auth': self._extract_mvpd_auth(
-                        url, video_id, 'fbc-fox', resource)
-                })
         m3u8_url = self._download_json(release_url, video_id)['playURL']
         formats = self._extract_m3u8_formats(
             m3u8_url, video_id, 'mp4',
             entry_protocol='m3u8_native', m3u8_id='hls')
         self._sort_formats(formats)
 
+        data = try_get(
+            video, lambda x: x['trackingData']['properties'], dict) or {}
+
         duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
             video.get('duration')) or parse_duration(video.get('duration'))
         timestamp = unified_timestamp(video.get('datePublished'))
@@ -117,7 +117,7 @@ class FOXIE(AdobePassIE):
             'description': video.get('description'),
             'duration': duration,
             'timestamp': timestamp,
-            'age_limit': parse_age_limit(rating),
+            'age_limit': parse_age_limit(video.get('contentRating')),
             'creator': creator,
             'series': series,
             'season_number': int_or_none(video.get('seasonNumber')),
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 165964ca0..ee12e2b47 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .fox import FOXIE
 from ..utils import (
     smuggle_url,
     url_basename,
@@ -58,3 +59,24 @@ class NationalGeographicVideoIE(InfoExtractor):
                 {'force_smil_url': True}),
             'id': guid,
         }
+
+
+class NationalGeographicTVIE(FOXIE):
+    _VALID_URL = r'https?://(?:www\.)?nationalgeographic\.com/tv/watch/(?P<id>[\da-fA-F]+)'
+    _TESTS = [{
+        'url': 'https://www.nationalgeographic.com/tv/watch/6a875e6e734b479beda26438c9f21138/',
+        'info_dict': {
+            'id': '6a875e6e734b479beda26438c9f21138',
+            'ext': 'mp4',
+            'title': 'Why Nat Geo? Valley of the Boom',
+            'description': 'The lives of prominent figures in the tech world, including their friendships, rivalries, victories and failures.',
+            'timestamp': 1542662458,
+            'upload_date': '20181119',
+            'age_limit': 14,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+    _HOME_PAGE_URL = 'https://www.nationalgeographic.com/tv/'
+    _API_KEY = '238bb0a0c2aba67922c48709ce0c06fd'

From a2d821d7112fb1423f99ddf309a843c80cc3be2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Jan 2019 23:33:09 +0700
Subject: [PATCH 374/510] [drtv] Improve extraction (closes #19039) + Add
 support for EncryptedUri videos + Extract more metadata * Fix subtitles
 extraction

---
 youtube_dl/extractor/drtv.py | 133 +++++++++++++++++++++++++++--------
 1 file changed, 102 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 8d63ca433..c5f211128 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,15 +1,25 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import binascii
+import hashlib
+import re
+
+
 from .common import InfoExtractor
+from ..aes import aes_cbc_decrypt
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
+    bytes_to_intlist,
     ExtractorError,
     int_or_none,
+    intlist_to_bytes,
     float_or_none,
     mimetype2ext,
-    parse_iso8601,
-    remove_end,
+    str_or_none,
+    unified_timestamp,
     update_url_query,
+    url_or_none,
 )
 
 
@@ -20,23 +30,31 @@ class DRTVIE(InfoExtractor):
     IE_NAME = 'drtv'
     _TESTS = [{
         'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
-        'md5': '7ae17b4e18eb5d29212f424a7511c184',
+        'md5': '25e659cccc9a2ed956110a299fdf5983',
         'info_dict': {
             'id': 'klassen-darlig-taber-10',
             'ext': 'mp4',
             'title': 'Klassen - Dårlig taber (10)',
             'description': 'md5:815fe1b7fa656ed80580f31e8b3c79aa',
-            'timestamp': 1471991907,
-            'upload_date': '20160823',
+            'timestamp': 1539085800,
+            'upload_date': '20181009',
             'duration': 606.84,
+            'series': 'Klassen',
+            'season': 'Klassen I',
+            'season_number': 1,
+            'season_id': 'urn:dr:mu:bundle:57d7e8216187a4031cfd6f6b',
+            'episode': 'Episode 10',
+            'episode_number': 10,
+            'release_year': 2016,
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         # embed
         'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
         'info_dict': {
-            'id': 'christiania-pusher-street-ryddes-drdkrjpo',
+            'id': 'urn:dr:mu:programcard:57c926176187a50a9c6e83c6',
             'ext': 'mp4',
-            'title': 'LIVE Christianias rydning af Pusher Street er i gang',
+            'title': 'christiania pusher street ryddes drdkrjpo',
             'description': 'md5:2a71898b15057e9b97334f61d04e6eb5',
             'timestamp': 1472800279,
             'upload_date': '20160902',
@@ -45,17 +63,18 @@ class DRTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         # with SignLanguage formats
         'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
         'info_dict': {
             'id': 'historien-om-danmark-stenalder',
             'ext': 'mp4',
-            'title': 'Historien om Danmark: Stenalder (1)',
+            'title': 'Historien om Danmark: Stenalder',
             'description': 'md5:8c66dcbc1669bbc6f873879880f37f2a',
-            'timestamp': 1490401996,
-            'upload_date': '20170325',
-            'duration': 3502.04,
+            'timestamp': 1546628400,
+            'upload_date': '20190104',
+            'duration': 3502.56,
             'formats': 'mincount:20',
         },
         'params': {
@@ -74,19 +93,26 @@ class DRTVIE(InfoExtractor):
 
         video_id = self._search_regex(
             (r'data-(?:material-identifier|episode-slug)="([^"]+)"',
-                r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
-            webpage, 'video id')
+             r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
+            webpage, 'video id', default=None)
+
+        if not video_id:
+            video_id = compat_urllib_parse_unquote(self._search_regex(
+                r'(urn(?:%3A|:)dr(?:%3A|:)mu(?:%3A|:)programcard(?:%3A|:)[\da-f]+)',
+                webpage, 'urn'))
 
         data = self._download_json(
             'https://www.dr.dk/mu-online/api/1.4/programcard/%s' % video_id,
             video_id, 'Downloading video JSON', query={'expanded': 'true'})
 
-        title = remove_end(self._og_search_title(
-            webpage, default=None), ' | TV | DR') or data['Title']
+        title = str_or_none(data.get('Title')) or re.sub(
+            r'\s*\|\s*(?:TV\s*\|\s*DR|DRTV)$', '',
+            self._og_search_title(webpage))
         description = self._og_search_description(
             webpage, default=None) or data.get('Description')
 
-        timestamp = parse_iso8601(data.get('CreatedTime'))
+        timestamp = unified_timestamp(
+            data.get('PrimaryBroadcastStartTime') or data.get('SortDateTime'))
 
         thumbnail = None
         duration = None
@@ -96,16 +122,51 @@ class DRTVIE(InfoExtractor):
         formats = []
         subtitles = {}
 
-        for asset in [data['PrimaryAsset']]:
+        assets = []
+        primary_asset = data.get('PrimaryAsset')
+        if isinstance(primary_asset, dict):
+            assets.append(primary_asset)
+        secondary_assets = data.get('SecondaryAssets')
+        if isinstance(secondary_assets, list):
+            for secondary_asset in secondary_assets:
+                if isinstance(secondary_asset, dict):
+                    assets.append(secondary_asset)
+
+        def hex_to_bytes(hex):
+            return binascii.a2b_hex(hex.encode('ascii'))
+
+        def decrypt_uri(e):
+            n = int(e[2:10], 16)
+            a = e[10 + n:]
+            data = bytes_to_intlist(hex_to_bytes(e[10:10 + n]))
+            key = bytes_to_intlist(hashlib.sha256(
+                ('%s:sRBzYNXBzkKgnjj8pGtkACch' % a).encode('utf-8')).digest())
+            iv = bytes_to_intlist(hex_to_bytes(a))
+            decrypted = aes_cbc_decrypt(data, key, iv)
+            return intlist_to_bytes(
+                decrypted[:-decrypted[-1]]).decode('utf-8').split('?')[0]
+
+        for asset in assets:
             kind = asset.get('Kind')
             if kind == 'Image':
-                thumbnail = asset.get('Uri')
+                thumbnail = url_or_none(asset.get('Uri'))
             elif kind in ('VideoResource', 'AudioResource'):
                 duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                 restricted_to_denmark = asset.get('RestrictedToDenmark')
                 asset_target = asset.get('Target')
                 for link in asset.get('Links', []):
                     uri = link.get('Uri')
+                    if not uri:
+                        encrypted_uri = link.get('EncryptedUri')
+                        if not encrypted_uri:
+                            continue
+                        try:
+                            uri = decrypt_uri(encrypted_uri)
+                        except Exception:
+                            self.report_warning(
+                                'Unable to decrypt EncryptedUri', video_id)
+                            continue
+                    uri = url_or_none(uri)
                     if not uri:
                         continue
                     target = link.get('Target')
@@ -139,19 +200,22 @@ class DRTVIE(InfoExtractor):
                             'vcodec': 'none' if kind == 'AudioResource' else None,
                             'preference': preference,
                         })
-                subtitles_list = asset.get('SubtitlesList')
-                if isinstance(subtitles_list, list):
-                    LANGS = {
-                        'Danish': 'da',
-                    }
-                    for subs in subtitles_list:
-                        if not subs.get('Uri'):
-                            continue
-                        lang = subs.get('Language') or 'da'
-                        subtitles.setdefault(LANGS.get(lang, lang), []).append({
-                            'url': subs['Uri'],
-                            'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
-                        })
+            subtitles_list = asset.get('SubtitlesList') or asset.get('Subtitleslist')
+            if isinstance(subtitles_list, list):
+                LANGS = {
+                    'Danish': 'da',
+                }
+                for subs in subtitles_list:
+                    if not isinstance(subs, dict):
+                        continue
+                    sub_uri = url_or_none(subs.get('Uri'))
+                    if not sub_uri:
+                        continue
+                    lang = subs.get('Language') or 'da'
+                    subtitles.setdefault(LANGS.get(lang, lang), []).append({
+                        'url': sub_uri,
+                        'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
+                    })
 
         if not formats and restricted_to_denmark:
             self.raise_geo_restricted(
@@ -169,6 +233,13 @@ class DRTVIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles,
+            'series': str_or_none(data.get('SeriesTitle')),
+            'season': str_or_none(data.get('SeasonTitle')),
+            'season_number': int_or_none(data.get('SeasonNumber')),
+            'season_id': str_or_none(data.get('SeasonUrn')),
+            'episode': str_or_none(data.get('EpisodeTitle')),
+            'episode_number': int_or_none(data.get('EpisodeNumber')),
+            'release_year': int_or_none(data.get('ProductionYear')),
         }
 
 

From 41cff90c41006b30213c7f676bd3920a1612b717 Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Mon, 28 Jan 2019 19:42:49 -0400
Subject: [PATCH 375/510] [yourporn] Fix extraction and extract duration
 (closes #18815, closes #18852)

change cdn to cdn4 for the video_url
---
 youtube_dl/extractor/yourporn.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py
index c8dc29bd8..01e5f0c0e 100644
--- a/youtube_dl/extractor/yourporn.py
+++ b/youtube_dl/extractor/yourporn.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import urljoin
+from ..utils import (
+    parse_duration,
+    urljoin
+)
 
 
 class YourPornIE(InfoExtractor):
@@ -27,17 +30,21 @@ class YourPornIE(InfoExtractor):
             self._search_regex(
                 r'data-vnfo=(["\'])(?P<data>{.+?})\1', webpage, 'data info',
                 group='data'),
-            video_id)[video_id]).replace('/cdn/', '/cdn3/')
+            video_id)[video_id]).replace('/cdn/', '/cdn4/')
 
         title = (self._search_regex(
             r'<[^>]+\bclass=["\']PostEditTA[^>]+>([^<]+)', webpage, 'title',
             default=None) or self._og_search_description(webpage)).strip()
+
         thumbnail = self._og_search_thumbnail(webpage)
 
+        duration = parse_duration(self._search_regex(r'duration:[^0-9]*([0-9:]+)',
+                                                     webpage, 'duration', default=None))
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
+            'duration': duration,
             'thumbnail': thumbnail,
             'age_limit': 18
         }

From 9868f1ab1853484d7a6c38cd6fa0d94a11914cae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Jan 2019 23:56:42 +0700
Subject: [PATCH 376/510] [yourporn] Improve (closes #19061)

---
 youtube_dl/extractor/yourporn.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py
index 01e5f0c0e..2c63f9752 100644
--- a/youtube_dl/extractor/yourporn.py
+++ b/youtube_dl/extractor/yourporn.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
-    urljoin
+    urljoin,
 )
 
 
@@ -17,7 +17,11 @@ class YourPornIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'md5:c9f43630bd968267672651ba905a7d35',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'age_limit': 18
+            'duration': 165,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
         },
     }
 
@@ -35,16 +39,16 @@ class YourPornIE(InfoExtractor):
         title = (self._search_regex(
             r'<[^>]+\bclass=["\']PostEditTA[^>]+>([^<]+)', webpage, 'title',
             default=None) or self._og_search_description(webpage)).strip()
-
         thumbnail = self._og_search_thumbnail(webpage)
+        duration = parse_duration(self._search_regex(
+            r'duration\s*:\s*<[^>]+>([\d:]+)', webpage, 'duration',
+            default=None))
 
-        duration = parse_duration(self._search_regex(r'duration:[^0-9]*([0-9:]+)',
-                                                     webpage, 'duration', default=None))
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
-            'duration': duration,
             'thumbnail': thumbnail,
-            'age_limit': 18
+            'duration': duration,
+            'age_limit': 18,
         }

From 5496754ae4c9097f37cfd9b307261cbbca438260 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Jan 2019 00:03:19 +0700
Subject: [PATCH 377/510] [fox] Remove unused imports

---
 youtube_dl/extractor/fox.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 568656542..0ffceeb7c 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -6,12 +6,10 @@ import uuid
 
 from .adobepass import AdobePassIE
 from ..compat import (
-    compat_HTTPError,
     compat_str,
     compat_urllib_parse_unquote,
 )
 from ..utils import (
-    ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_duration,

From ca01e5f9039dd6c0d5abff5c7139f82c5d1dfba7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Jan 2019 00:05:32 +0700
Subject: [PATCH 378/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d94fe36ec..8f5343b23 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Do not copy Apple TV chapter tracks while embedding
+  subtitles (#19024, #19042)
+* [postprocessor/ffmpeg] Disable "Last message repeated" messages (#19025)
+
+Extractors
+* [yourporn] Fix extraction and extract duration (#18815, #18852, #19061)
+* [drtv] Improve extraction (#19039)
+    + Add support for EncryptedUri videos
+    + Extract more metadata
+    * Fix subtitles extraction
++ [fox] Add support for locked videos using cookies (#19060)
+* [fox] Fix extraction for free videos (#19060)
++ [zattoo] Add support for tv.salt.ch (#19059)
+
+
 version 2019.01.27
 
 Core

From 1063b4c7073ce056f694b1690dd5d5a1a06fb347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Jan 2019 00:08:39 +0700
Subject: [PATCH 379/510] release 2019.01.30

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f529e3f4b..3944a4a38 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.27*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.27**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.30*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.30**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.27
+[debug] youtube-dl version 2019.01.30
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 8f5343b23..745fffeaa 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.30
 
 Core
 * [postprocessor/ffmpeg] Do not copy Apple TV chapter tracks while embedding
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6377bf815..2918520c3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -546,6 +546,7 @@
  - **MyVisionTV**
  - **n-tv.de**
  - **natgeo:video**
+ - **NationalGeographicTV**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -776,6 +777,7 @@
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
  - **SAKTV**
+ - **SaltTV**
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ec89cfc64..97818d0c7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.27'
+__version__ = '2019.01.30'

From ce52c7c111602f41d7f9c498f2915fd255ba2eab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Jan 2019 06:15:23 +0700
Subject: [PATCH 380/510] [postprocessor/ffmpeg] Fix avconv processing broken
 in #19025 (closes #19067)

---
 youtube_dl/postprocessor/ffmpeg.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 88b9ae9be..5bcb00ac0 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -217,11 +217,13 @@ class FFmpegPostProcessor(PostProcessor):
                 encodeArgument('-i'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)
             ])
-        cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
-               [encodeArgument('-loglevel'), encodeArgument('repeat+info')] +
-               files_cmd +
-               [encodeArgument(o) for o in opts] +
-               [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
+        cmd = [encodeFilename(self.executable, True), encodeArgument('-y')]
+        # avconv does not have repeat option
+        if self.basename == 'ffmpeg':
+            cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')]
+        cmd += (files_cmd +
+                [encodeArgument(o) for o in opts] +
+                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
         if self._downloader.params.get('verbose', False):
             self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))

From c2a0fe2ea7422c437a27c8fac57c7e865517354b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Jan 2019 06:17:25 +0700
Subject: [PATCH 381/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 745fffeaa..e6de6ca03 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Fix avconv processing broken in #19025 (#19067)
+
+
 version 2019.01.30
 
 Core

From 7b0f9df23d9842ddb2a545a0ceaf594daa0e12ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Jan 2019 06:19:36 +0700
Subject: [PATCH 382/510] release 2019.01.30.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3944a4a38..423a08e4d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.30*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.30**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.30.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.30.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.30
+[debug] youtube-dl version 2019.01.30.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e6de6ca03..4872cd9fc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.01.30.1
 
 Core
 * [postprocessor/ffmpeg] Fix avconv processing broken in #19025 (#19067)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 97818d0c7..be3bbdd73 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.30'
+__version__ = '2019.01.30.1'

From 645c4885cf38ecb244412dffda2760f4c0e72033 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Jan 2019 14:43:44 +0100
Subject: [PATCH 383/510] [crackle] authorize media detail request(closes
 #16931)

---
 youtube_dl/extractor/crackle.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index f73ef6b63..49bf3a4f9 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals, division
 
+import hashlib
+import hmac
 import re
+import time
 
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
@@ -74,13 +77,16 @@ class CrackleIE(InfoExtractor):
 
         for country in countries:
             try:
+                # Authorization generation algorithm is reverse engineered from:
+                # https://www.sonycrackle.com/static/js/main.ea93451f.chunk.js
+                media_detail_url = 'https://web-api-us.crackle.com/Service.svc/details/media/%s/%s?disableProtocols=true' % (video_id, country)
+                timestamp = time.strftime('%Y%m%d%H%M', time.gmtime())
+                h = hmac.new(b'IGSLUQCBDFHEOIFM', '|'.join([media_detail_url, timestamp]).encode(), hashlib.sha1).hexdigest().upper()
                 media = self._download_json(
-                    'https://web-api-us.crackle.com/Service.svc/details/media/%s/%s'
-                    % (video_id, country), video_id,
-                    'Downloading media JSON as %s' % country,
-                    'Unable to download media JSON', query={
-                        'disableProtocols': 'true',
-                        'format': 'json'
+                    media_detail_url, video_id, 'Downloading media JSON as %s' % country,
+                    'Unable to download media JSON', headers={
+                        'Accept': 'application/json',
+                        'Authorization': '|'.join([h, timestamp, '117', '1']),
                     })
             except ExtractorError as e:
                 # 401 means geo restriction, trying next country

From 15e832ff2a1bee42b299d3498439cf789c16fffa Mon Sep 17 00:00:00 2001
From: Batuhan's Unmaintained Account <batuhanosmantaskaya@gmail.com>
Date: Wed, 30 Jan 2019 19:39:02 +0300
Subject: [PATCH 384/510] [openload] Add support for oload.info

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b713e78b8..747aa298a 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -249,7 +249,7 @@ class OpenloadIE(InfoExtractor):
                             (?:www\.)?
                             (?:
                                 openload\.(?:co|io|link)|
-                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club)
+                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info)
                             )
                         )/
                         (?:f|embed)/
@@ -337,6 +337,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.club/f/Nr1L-aZ2dbQ',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.info/f/5NEAbI2BDSk',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From 9613e14a92429046b162145dfd40dee5795ca409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Jan 2019 00:15:45 +0700
Subject: [PATCH 385/510] [openload] Add support for openload.pw and oload.pw
 (closes #18930)

---
 youtube_dl/extractor/openload.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 747aa298a..a2ae25272 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -248,8 +248,8 @@ class OpenloadIE(InfoExtractor):
                         (?P<host>
                             (?:www\.)?
                             (?:
-                                openload\.(?:co|io|link)|
-                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info)
+                                openload\.(?:co|io|link|pw)|
+                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw)
                             )
                         )/
                         (?:f|embed)/
@@ -340,6 +340,12 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.info/f/5NEAbI2BDSk',
         'only_matching': True,
+    }, {
+        'url': 'https://openload.pw/f/WyKgK8s94N0',
+        'only_matching': True,
+    }, {
+        'url': 'https://oload.pw/f/WyKgK8s94N0',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From 49fe4175ae165527a0b06b8d97cdc85d83041fef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Feb 2019 01:48:10 +0700
Subject: [PATCH 386/510] [drtv] Improve preference (closes #19079)

---
 youtube_dl/extractor/drtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index c5f211128..0c7e350f0 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -171,10 +171,13 @@ class DRTVIE(InfoExtractor):
                         continue
                     target = link.get('Target')
                     format_id = target or ''
-                    preference = None
-                    if asset_target in ('SpokenSubtitles', 'SignLanguage'):
+                    if asset_target in ('SpokenSubtitles', 'SignLanguage', 'VisuallyInterpreted'):
                         preference = -1
                         format_id += '-%s' % asset_target
+                    elif asset_target == 'Default':
+                        preference = 1
+                    else:
+                        preference = None
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',

From 3ef2da2d21061bd44df0b0a0d27e82a365209662 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Feb 2019 04:00:29 +0700
Subject: [PATCH 387/510] [soundcloud] Fix paged playlists extraction, add
 support for albums and update client id

---
 youtube_dl/extractor/soundcloud.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 81c81c8d5..030840fd8 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -34,7 +34,7 @@ class SoundcloudIE(InfoExtractor):
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?!stations/track)
                             (?P<uploader>[\w\d-]+)/
-                            (?!(?:tracks|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
+                            (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -157,7 +157,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = 'LvWovRaJZlWCHql0bISuum8Bd2KX79mb'
+    _CLIENT_ID = 'NmW1FlPaiL94ueEu7oziOWjYEzZzQDcK'
 
     @staticmethod
     def _extract_urls(webpage):
@@ -368,7 +368,6 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
 
 
 class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
-    _API_BASE = 'https://api.soundcloud.com'
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _extract_playlist(self, base_url, playlist_id, playlist_title):
@@ -389,8 +388,12 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
                 next_href, playlist_id, 'Downloading track page %s' % (i + 1))
 
             collection = response['collection']
-            if not collection:
-                break
+
+            if not isinstance(collection, list):
+                collection = []
+
+            # Empty collection may be returned, in this case we proceed
+            # straight to next_href
 
             def resolve_permalink_url(candidates):
                 for cand in candidates:
@@ -429,7 +432,7 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
                             (?:(?:www|m)\.)?soundcloud\.com/
                             (?P<user>[^/]+)
                             (?:/
-                                (?P<rsrc>tracks|sets|reposts|likes|spotlight)
+                                (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
                             )?
                             /?(?:[?#].*)?$
                     '''
@@ -476,13 +479,17 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
             'title': 'Grynpyret (Spotlight)',
         },
         'playlist_mincount': 1,
+    }, {
+        'url': 'https://soundcloud.com/soft-cell-official/albums',
+        'only_matching': True,
     }]
 
     _BASE_URL_MAP = {
-        'all': '%s/profile/soundcloud:users:%%s' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
-        'tracks': '%s/users/%%s/tracks' % SoundcloudPagedPlaylistBaseIE._API_BASE,
+        'all': '%s/stream/users/%%s' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'tracks': '%s/users/%%s/tracks' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'albums': '%s/users/%%s/albums' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
         'sets': '%s/users/%%s/playlists' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
-        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'reposts': '%s/stream/users/%%s/reposts' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
         'likes': '%s/users/%%s/likes' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
         'spotlight': '%s/users/%%s/spotlight' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
     }
@@ -490,6 +497,7 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
     _TITLE_MAP = {
         'all': 'All',
         'tracks': 'Tracks',
+        'albums': 'Albums',
         'sets': 'Playlists',
         'reposts': 'Reposts',
         'likes': 'Likes',

From b6423e6ca215e1583e013cf7b2c1faf8d3dcace7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Feb 2019 04:11:32 +0700
Subject: [PATCH 388/510] [soundcloud:user] Update tests

---
 youtube_dl/extractor/soundcloud.py | 48 ++++++++++++++++--------------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 030840fd8..13463ae4f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -438,40 +438,47 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
                     '''
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
-        'url': 'https://soundcloud.com/the-akashic-chronicler',
+        'url': 'https://soundcloud.com/soft-cell-official',
         'info_dict': {
-            'id': '114582580',
-            'title': 'The Akashic Chronicler (All)',
+            'id': '207965082',
+            'title': 'Soft Cell (All)',
         },
-        'playlist_mincount': 74,
+        'playlist_mincount': 28,
     }, {
-        'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
+        'url': 'https://soundcloud.com/soft-cell-official/tracks',
         'info_dict': {
-            'id': '114582580',
-            'title': 'The Akashic Chronicler (Tracks)',
+            'id': '207965082',
+            'title': 'Soft Cell (Tracks)',
         },
-        'playlist_mincount': 37,
+        'playlist_mincount': 27,
     }, {
-        'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
+        'url': 'https://soundcloud.com/soft-cell-official/albums',
         'info_dict': {
-            'id': '114582580',
-            'title': 'The Akashic Chronicler (Playlists)',
+            'id': '207965082',
+            'title': 'Soft Cell (Albums)',
+        },
+        'playlist_mincount': 1,
+    }, {
+        'url': 'https://soundcloud.com/jcv246/sets',
+        'info_dict': {
+            'id': '12982173',
+            'title': 'Jordi / cv (Playlists)',
         },
         'playlist_mincount': 2,
     }, {
-        'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
+        'url': 'https://soundcloud.com/jcv246/reposts',
         'info_dict': {
-            'id': '114582580',
-            'title': 'The Akashic Chronicler (Reposts)',
+            'id': '12982173',
+            'title': 'Jordi / cv (Reposts)',
         },
-        'playlist_mincount': 7,
+        'playlist_mincount': 6,
     }, {
-        'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
+        'url': 'https://soundcloud.com/clalberg/likes',
         'info_dict': {
-            'id': '114582580',
-            'title': 'The Akashic Chronicler (Likes)',
+            'id': '11817582',
+            'title': 'clalberg (Likes)',
         },
-        'playlist_mincount': 321,
+        'playlist_mincount': 5,
     }, {
         'url': 'https://soundcloud.com/grynpyret/spotlight',
         'info_dict': {
@@ -479,9 +486,6 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
             'title': 'Grynpyret (Spotlight)',
         },
         'playlist_mincount': 1,
-    }, {
-        'url': 'https://soundcloud.com/soft-cell-official/albums',
-        'only_matching': True,
     }]
 
     _BASE_URL_MAP = {

From e9fef7ee4e666b60bc7a757391f16e2be76f6cbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Feb 2019 05:44:31 +0700
Subject: [PATCH 389/510] [YoutubeDL] Fallback to ie_key of matching extractor
 while making download archive id when no explicit ie_key is provided (#19022)

---
 youtube_dl/YoutubeDL.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 80ed8d7e5..c168415ce 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2060,15 +2060,21 @@ class YoutubeDL(object):
                         self.report_warning('Unable to remove downloaded original file')
 
     def _make_archive_id(self, info_dict):
+        video_id = info_dict.get('id')
+        if not video_id:
+            return
         # Future-proof against any change in case
         # and backwards compatibility with prior versions
-        extractor = info_dict.get('extractor_key')
+        extractor = info_dict.get('extractor_key') or info_dict.get('ie_key')  # key in a playlist
         if extractor is None:
-            if 'id' in info_dict:
-                extractor = info_dict.get('ie_key')  # key in a playlist
-        if extractor is None:
-            return None  # Incomplete video information
-        return extractor.lower() + ' ' + info_dict['id']
+            # Try to find matching extractor for the URL and take its ie_key
+            for ie in self._ies:
+                if ie.suitable(info_dict['url']):
+                    extractor = ie.ie_key()
+                    break
+            else:
+                return
+        return extractor.lower() + ' ' + video_id
 
     def in_download_archive(self, info_dict):
         fn = self.params.get('download_archive')
@@ -2076,7 +2082,7 @@ class YoutubeDL(object):
             return False
 
         vid_id = self._make_archive_id(info_dict)
-        if vid_id is None:
+        if not vid_id:
             return False  # Incomplete video information
 
         try:

From b9bc1cff721b6f63e733c6ababeec45b92f1484b Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Fri, 1 Feb 2019 19:04:00 -0400
Subject: [PATCH 390/510] [drtuber] Extract duration

---
 youtube_dl/extractor/drtuber.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 5c41c8022..2baea585b 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -4,7 +4,9 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     NO_DEFAULT,
+    parse_duration,
     str_to_int,
 )
 
@@ -65,6 +67,9 @@ class DrTuberIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        duration = int_or_none(video_data.get('duration')) or parse_duration(
+            video_data.get('duration_format'))
+
         title = self._html_search_regex(
             (r'<h1[^>]+class=["\']title[^>]+>([^<]+)',
              r'<title>([^<]+)\s*@\s+DrTuber',
@@ -103,4 +108,5 @@ class DrTuberIE(InfoExtractor):
             'comment_count': comment_count,
             'categories': categories,
             'age_limit': self._rta_search(webpage),
+            'duration': duration,
         }

From 6cc6e0c34d0f67747be7bac91690820f47b26acb Mon Sep 17 00:00:00 2001
From: Cory Hall <corydantehall@gmail.com>
Date: Thu, 31 Jan 2019 20:51:37 -0500
Subject: [PATCH 391/510] [soundcloud:pagedplaylist] Add ie and title to
 entries (#19022)

rel: https://github.com/rg3/youtube-dl/issues/19022
---
 youtube_dl/extractor/soundcloud.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 13463ae4f..1c8d3c53b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -395,18 +395,20 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
             # Empty collection may be returned, in this case we proceed
             # straight to next_href
 
-            def resolve_permalink_url(candidates):
-                for cand in candidates:
+            def append_url_result(entries, item):
+                for cand in (item, item.get('track'), item.get('playlist')):
                     if isinstance(cand, dict):
                         permalink_url = cand.get('permalink_url')
-                        entry_id = self._extract_id(cand)
                         if permalink_url and permalink_url.startswith('http'):
-                            return permalink_url, entry_id
+                            return entries.append(
+                                self.url_result(
+                                    permalink_url,
+                                    ie=SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
+                                    video_id=self._extract_id(cand),
+                                    video_title=cand.get('title')))
 
             for e in collection:
-                permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
-                if permalink_url:
-                    entries.append(self.url_result(permalink_url, video_id=entry_id))
+                append_url_result(entries, e)
 
             next_href = response.get('next_href')
             if not next_href:

From 7c5307f4c4e91ef6551d70cd844b93fbdc5c3cf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Feb 2019 23:40:06 +0700
Subject: [PATCH 392/510] [soundcloud:pagedplaylist] Improve (closes #19086)

---
 youtube_dl/extractor/soundcloud.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 1c8d3c53b..5536e7851 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -18,6 +18,7 @@ from ..utils import (
     int_or_none,
     unified_strdate,
     update_url_query,
+    url_or_none,
 )
 
 
@@ -395,20 +396,23 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
             # Empty collection may be returned, in this case we proceed
             # straight to next_href
 
-            def append_url_result(entries, item):
-                for cand in (item, item.get('track'), item.get('playlist')):
-                    if isinstance(cand, dict):
-                        permalink_url = cand.get('permalink_url')
-                        if permalink_url and permalink_url.startswith('http'):
-                            return entries.append(
-                                self.url_result(
-                                    permalink_url,
-                                    ie=SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
-                                    video_id=self._extract_id(cand),
-                                    video_title=cand.get('title')))
+            def resolve_entry(candidates):
+                for cand in candidates:
+                    if not isinstance(cand, dict):
+                        continue
+                    permalink_url = url_or_none(cand.get('permalink_url'))
+                    if not permalink_url:
+                        continue
+                    return self.url_result(
+                        permalink_url,
+                        ie=SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
+                        video_id=self._extract_id(cand),
+                        video_title=cand.get('title'))
 
             for e in collection:
-                append_url_result(entries, e)
+                entry = resolve_entry((e, e.get('track'), e.get('playlist')))
+                if entry:
+                    entries.append(entry)
 
             next_href = response.get('next_href')
             if not next_href:

From 0efcb5a2fe0c3024d3e5affe74b3d0d416413ffa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Feb 2019 00:33:45 +0700
Subject: [PATCH 393/510] [vporn] Remove extractor (closes #16276) Handled by
 generic extractor

---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/vporn.py      | 123 -----------------------------
 2 files changed, 124 deletions(-)
 delete mode 100644 youtube_dl/extractor/vporn.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b40be42e6..693c16e49 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1362,7 +1362,6 @@ from .voxmedia import (
     VoxMediaVolumeIE,
     VoxMediaIE,
 )
-from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
 from .vrv import (
diff --git a/youtube_dl/extractor/vporn.py b/youtube_dl/extractor/vporn.py
deleted file mode 100644
index 858ac9e71..000000000
--- a/youtube_dl/extractor/vporn.py
+++ /dev/null
@@ -1,123 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    parse_duration,
-    str_to_int,
-    urljoin,
-)
-
-
-class VpornIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vporn\.com/[^/]+/(?P<display_id>[^/]+)/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.vporn.com/masturbation/violet-on-her-th-birthday/497944/',
-            'md5': 'facf37c1b86546fa0208058546842c55',
-            'info_dict': {
-                'id': '497944',
-                'display_id': 'violet-on-her-th-birthday',
-                'ext': 'mp4',
-                'title': 'Violet on her 19th birthday',
-                'description': 'Violet dances in front of the camera which is sure to get you horny.',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'uploader': 'kileyGrope',
-                'categories': ['Masturbation', 'Teen'],
-                'duration': 393,
-                'age_limit': 18,
-                'view_count': int,
-            },
-            'skip': 'video removed',
-        },
-        {
-            'url': 'http://www.vporn.com/female/hana-shower/523564/',
-            'md5': 'ced35a4656198a1664cf2cda1575a25f',
-            'info_dict': {
-                'id': '523564',
-                'display_id': 'hana-shower',
-                'ext': 'mp4',
-                'title': 'Hana Shower',
-                'description': 'Hana showers at the bathroom.',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'uploader': 'Hmmmmm',
-                'categories': ['Big Boobs', 'Erotic', 'Teen', 'Female', '720p'],
-                'duration': 588,
-                'age_limit': 18,
-                'view_count': int,
-            }
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        errmsg = 'This video has been deleted due to Copyright Infringement or by the account owner!'
-        if errmsg in webpage:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
-
-        title = self._html_search_regex(
-            r'videoname\s*=\s*\'([^\']+)\'', webpage, 'title').strip()
-        description = self._html_search_regex(
-            r'class="(?:descr|description_txt)">(.*?)</div>',
-            webpage, 'description', fatal=False)
-        thumbnail = urljoin('http://www.vporn.com', self._html_search_regex(
-            r'flashvars\.imageUrl\s*=\s*"([^"]+)"', webpage, 'description',
-            default=None))
-
-        uploader = self._html_search_regex(
-            r'(?s)Uploaded by:.*?<a href="/user/[^"]+"[^>]*>(.+?)</a>',
-            webpage, 'uploader', fatal=False)
-
-        categories = re.findall(r'<a href="/cat/[^"]+"[^>]*>([^<]+)</a>', webpage)
-
-        duration = parse_duration(self._search_regex(
-            r'Runtime:\s*</span>\s*(\d+ min \d+ sec)',
-            webpage, 'duration', fatal=False))
-
-        view_count = str_to_int(self._search_regex(
-            r'class="views">([\d,\.]+) [Vv]iews<',
-            webpage, 'view count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r"'Comments \(([\d,\.]+)\)'",
-            webpage, 'comment count', default=None))
-
-        formats = []
-
-        for video in re.findall(r'flashvars\.videoUrl([^=]+?)\s*=\s*"(https?://[^"]+)"', webpage):
-            video_url = video[1]
-            fmt = {
-                'url': video_url,
-                'format_id': video[0],
-            }
-            m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)_(?P<vbr>\d+)k\.mp4$', video_url)
-            if m:
-                fmt.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
-                    'vbr': int(m.group('vbr')),
-                })
-            formats.append(fmt)
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'categories': categories,
-            'duration': duration,
-            'view_count': view_count,
-            'comment_count': comment_count,
-            'age_limit': 18,
-            'formats': formats,
-        }

From eecf788b90fa4d49567c714f5a613fdd2b6e2507 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Feb 2019 09:10:09 +0100
Subject: [PATCH 394/510] [teachable] add support for courses.workitdaily.com
 (closes #18871)

---
 youtube_dl/extractor/teachable.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/teachable.py b/youtube_dl/extractor/teachable.py
index 47ac95ee8..c1a9deafe 100644
--- a/youtube_dl/extractor/teachable.py
+++ b/youtube_dl/extractor/teachable.py
@@ -27,6 +27,7 @@ class TeachableBaseIE(InfoExtractor):
         'market.saleshacker.com': 'saleshacker',
         'learnability.org': 'learnability',
         'edurila.com': 'edurila',
+        'courses.workitdaily.com': 'workitdaily',
     }
 
     _VALID_URL_SUB_TUPLE = (_URL_PREFIX, '|'.join(re.escape(site) for site in _SITES.keys()))

From 07fbfef1c7e36b25dd7098be73ab76b87378a015 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Feb 2019 12:10:41 +0100
Subject: [PATCH 395/510] [radiocanada] switch to the new media requests(closes
 #19115)

---
 youtube_dl/extractor/radiocanada.py | 133 ++++++++--------------------
 1 file changed, 39 insertions(+), 94 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 302f67d96..58e294892 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -5,14 +5,10 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    xpath_text,
-    find_xpath_attr,
     determine_ext,
+    ExtractorError,
     int_or_none,
     unified_strdate,
-    xpath_element,
-    ExtractorError,
-    determine_protocol,
     unsmuggle_url,
 )
 
@@ -61,107 +57,53 @@ class RadioCanadaIE(InfoExtractor):
             'only_matching': True,
         }
     ]
+    _GEO_COUNTRIES = ['CA']
+
+    def _call_api(self, path, video_id, app_code, query):
+        query.update({
+            'appCode': app_code,
+            'idMedia': video_id,
+            'output': 'json',
+        })
+        return self._download_json(
+            'https://services.radio-canada.ca/media/' + path, video_id, headers={
+                'Authorization': 'Client-Key 773aea60-0e80-41bb-9c7f-e6d7c3ad17fb'
+            }, query=query)
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
         app_code, video_id = re.match(self._VALID_URL, url).groups()
 
-        metadata = self._download_xml(
-            'http://api.radio-canada.ca/metaMedia/v1/index.ashx',
-            video_id, note='Downloading metadata XML', query={
-                'appCode': app_code,
-                'idMedia': video_id,
-            })
+        metas = self._call_api('meta/v1/index.ashx', video_id, app_code, {})['Metas']
 
         def get_meta(name):
-            el = find_xpath_attr(metadata, './/Meta', 'name', name)
-            return el.text if el is not None else None
+            for meta in metas:
+                if meta.get('name') == name:
+                    text = meta.get('text')
+                    if text:
+                        return text
 
         # protectionType does not necessarily mean the video is DRM protected (see
         # https://github.com/rg3/youtube-dl/pull/18609).
         if get_meta('protectionType'):
             self.report_warning('This video is probably DRM protected.')
 
-        device_types = ['ipad']
-        if not smuggled_data:
-            device_types.append('flash')
-            device_types.append('android')
-
-        formats = []
-        error = None
-        # TODO: extract f4m formats
-        # f4m formats can be extracted using flashhd device_type but they produce unplayable file
-        for device_type in device_types:
-            validation_url = 'http://api.radio-canada.ca/validationMedia/v1/Validation.ashx'
-            query = {
-                'appCode': app_code,
-                'idMedia': video_id,
-                'connectionType': 'broadband',
-                'multibitrate': 'true',
-                'deviceType': device_type,
-            }
-            if smuggled_data:
-                validation_url = 'https://services.radio-canada.ca/media/validation/v2/'
-                query.update(smuggled_data)
-            else:
-                query.update({
-                    # paysJ391wsHjbOJwvCs26toz and bypasslock are used to bypass geo-restriction
-                    'paysJ391wsHjbOJwvCs26toz': 'CA',
-                    'bypasslock': 'NZt5K62gRqfc',
-                })
-            v_data = self._download_xml(validation_url, video_id, note='Downloading %s XML' % device_type, query=query, fatal=False)
-            v_url = xpath_text(v_data, 'url')
-            if not v_url:
-                continue
-            if v_url == 'null':
-                error = xpath_text(v_data, 'message')
-                continue
-            ext = determine_ext(v_url)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    v_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    v_url, video_id, f4m_id='hds', fatal=False))
-            else:
-                ext = determine_ext(v_url)
-                bitrates = xpath_element(v_data, 'bitrates')
-                for url_e in bitrates.findall('url'):
-                    tbr = int_or_none(url_e.get('bitrate'))
-                    if not tbr:
-                        continue
-                    f_url = re.sub(r'\d+\.%s' % ext, '%d.%s' % (tbr, ext), v_url)
-                    protocol = determine_protocol({'url': f_url})
-                    f = {
-                        'format_id': '%s-%d' % (protocol, tbr),
-                        'url': f_url,
-                        'ext': 'flv' if protocol == 'rtmp' else ext,
-                        'protocol': protocol,
-                        'width': int_or_none(url_e.get('width')),
-                        'height': int_or_none(url_e.get('height')),
-                        'tbr': tbr,
-                    }
-                    mobj = re.match(r'(?P<url>rtmp://[^/]+/[^/]+)/(?P<playpath>[^?]+)(?P<auth>\?.+)', f_url)
-                    if mobj:
-                        f.update({
-                            'url': mobj.group('url') + mobj.group('auth'),
-                            'play_path': mobj.group('playpath'),
-                        })
-                    formats.append(f)
-                    if protocol == 'rtsp':
-                        base_url = self._search_regex(
-                            r'rtsp://([^?]+)', f_url, 'base url', default=None)
-                        if base_url:
-                            base_url = 'http://' + base_url
-                            formats.extend(self._extract_m3u8_formats(
-                                base_url + '/playlist.m3u8', video_id, 'mp4',
-                                'm3u8_native', m3u8_id='hls', fatal=False))
-                            formats.extend(self._extract_f4m_formats(
-                                base_url + '/manifest.f4m', video_id,
-                                f4m_id='hds', fatal=False))
-        if not formats and error:
+        query = {
+            'connectionType': 'hd',
+            'deviceType': 'ipad',
+            'multibitrate': 'true',
+        }
+        if smuggled_data:
+            query.update(smuggled_data)
+        v_data = self._call_api('validation/v2/', video_id, app_code, query)
+        v_url = v_data.get('url')
+        if not v_url:
+            error = v_data['message']
+            if error == "Le contenu sélectionné n'est pas disponible dans votre pays":
+                raise self.raise_geo_restricted(error, self._GEO_COUNTRIES)
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, error), expected=True)
+        formats = self._extract_m3u8_formats(v_url, video_id, 'mp4')
         self._sort_formats(formats)
 
         subtitles = {}
@@ -189,8 +131,8 @@ class RadioCanadaIE(InfoExtractor):
 
 class RadioCanadaAudioVideoIE(InfoExtractor):
     'radiocanada:audiovideo'
-    _VALID_URL = r'https?://ici\.radio-canada\.ca/audio-video/media-(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://ici\.radio-canada\.ca/([^/]+/)*media-(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://ici.radio-canada.ca/audio-video/media-7527184/barack-obama-au-vietnam',
         'info_dict': {
             'id': '7527184',
@@ -203,7 +145,10 @@ class RadioCanadaAudioVideoIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'https://ici.radio-canada.ca/info/videos/media-7527184/barack-obama-au-vietnam',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         return self.url_result('radiocanada:medianet:%s' % self._match_id(url))

From 70c3ee13671798d7e3b80ea9be863ef73bd08653 Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Mon, 4 Feb 2019 13:06:04 -0400
Subject: [PATCH 396/510] [pornhd] Extract like count

---
 youtube_dl/extractor/pornhd.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index b52879c7a..a079cd32a 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -23,6 +23,7 @@ class PornHdIE(InfoExtractor):
             'description': 'md5:3748420395e03e31ac96857a8f125b2b',
             'thumbnail': r're:^https?://.*\.jpg',
             'view_count': int,
+            'like_count': int,
             'age_limit': 18,
         }
     }, {
@@ -37,6 +38,7 @@ class PornHdIE(InfoExtractor):
             'description': 'md5:8ff0523848ac2b8f9b065ba781ccf294',
             'thumbnail': r're:^https?://.*\.jpg',
             'view_count': int,
+            'like_count': int,
             'age_limit': 18,
         },
         'skip': 'Not available anymore',
@@ -85,6 +87,11 @@ class PornHdIE(InfoExtractor):
             r"poster'?\s*:\s*([\"'])(?P<url>(?:(?!\1).)+)\1", webpage,
             'thumbnail', fatal=False, group='url')
 
+        like_count = int_or_none(self._search_regex(
+            (r'(\d+)\s*</11[^>]+>(?:&nbsp;|\s)*\blikes',
+             r'class=["\']save-count["\'][^>]*>\s*(\d+)'),
+            webpage, 'like count', fatal=False))
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -92,6 +99,7 @@ class PornHdIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'view_count': view_count,
+            'like_count': like_count,
             'formats': formats,
             'age_limit': 18,
         }

From 48fb963b2f9495922a4acf751608167cbc273693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Feb 2019 00:07:37 +0700
Subject: [PATCH 397/510] [pornhd] Fix formats extraction

---
 youtube_dl/extractor/pornhd.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index a079cd32a..27d65d4b9 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -4,9 +4,11 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     js_to_json,
+    urljoin,
 )
 
 
@@ -14,7 +16,7 @@ class PornHdIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
     _TESTS = [{
         'url': 'http://www.pornhd.com/videos/9864/selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
-        'md5': 'c8b964b1f0a4b5f7f28ae3a5c9f86ad5',
+        'md5': '87f1540746c1d32ec7a2305c12b96b25',
         'info_dict': {
             'id': '9864',
             'display_id': 'selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
@@ -67,12 +69,14 @@ class PornHdIE(InfoExtractor):
 
         formats = []
         for format_id, video_url in sources.items():
+            video_url = urljoin(url, video_url)
             if not video_url:
                 continue
             height = int_or_none(self._search_regex(
                 r'^(\d+)[pP]', format_id, 'height', default=None))
             formats.append({
                 'url': video_url,
+                'ext': determine_ext(video_url, 'mp4'),
                 'format_id': format_id,
                 'height': height,
             })

From d2d970d07ec82f648b62bff8b15ac0b57d0d0496 Mon Sep 17 00:00:00 2001
From: JChris246 <chris.401@live.com>
Date: Mon, 4 Feb 2019 13:33:54 -0400
Subject: [PATCH 398/510] [pornhub] Fix tags and categories extraction (closes
 #13720)

---
 youtube_dl/extractor/pornhub.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index be93d5d48..428324ef0 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -16,7 +16,6 @@ from .openload import PhantomJSwrapper
 from ..utils import (
     ExtractorError,
     int_or_none,
-    js_to_json,
     orderedSet,
     remove_quotes,
     str_to_int,
@@ -303,14 +302,17 @@ class PornHubIE(PornHubBaseIE):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        page_params = self._parse_json(self._search_regex(
-            r'page_params\.zoneDetails\[([\'"])[^\'"]+\1\]\s*=\s*(?P<data>{[^}]+})',
-            webpage, 'page parameters', group='data', default='{}'),
-            video_id, transform_source=js_to_json, fatal=False)
-        tags = categories = None
-        if page_params:
-            tags = page_params.get('tags', '').split(',')
-            categories = page_params.get('categories', '').split(',')
+        def _get_items(class_name):
+            div = self._search_regex(
+                r'<div class="' + class_name + '">([\S\s]+?)</div>',
+                webpage, class_name, default=None)
+            if div:
+                return [a for a in re.findall(r'<a href=[^>]+>([^<]+)', div)]
+            else:
+                return None
+
+        categories = _get_items('categoriesWrapper')
+        tags = _get_items('tagsWrapper')
 
         return {
             'id': video_id,

From 5dda1edef93d94c9a49672f905df0c49c75c5739 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Feb 2019 23:06:55 +0700
Subject: [PATCH 399/510] [pornhub] Improve and simplify (closes #19135)

---
 youtube_dl/extractor/pornhub.py | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 428324ef0..641083da7 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -302,17 +302,12 @@ class PornHubIE(PornHubBaseIE):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        def _get_items(class_name):
+        def extract_list(meta_key):
             div = self._search_regex(
-                r'<div class="' + class_name + '">([\S\s]+?)</div>',
-                webpage, class_name, default=None)
+                r'(?s)<div[^>]+\bclass=["\'].*?\b%sWrapper[^>]*>(.+?)</div>'
+                % meta_key, webpage, meta_key, default=None)
             if div:
-                return [a for a in re.findall(r'<a href=[^>]+>([^<]+)', div)]
-            else:
-                return None
-
-        categories = _get_items('categoriesWrapper')
-        tags = _get_items('tagsWrapper')
+                return re.findall(r'<a[^>]+\bhref=[^>]+>([^<]+)', div)
 
         return {
             'id': video_id,
@@ -327,8 +322,8 @@ class PornHubIE(PornHubBaseIE):
             'comment_count': comment_count,
             'formats': formats,
             'age_limit': 18,
-            'tags': tags,
-            'categories': categories,
+            'tags': extract_list('tags'),
+            'categories': extract_list('categories'),
             'subtitles': subtitles,
         }
 

From 8fecc7353df35f6cac305c04a4e203fb2bbb4827 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Feb 2019 13:59:12 +0100
Subject: [PATCH 400/510] [toutv] fix authentication(closes #16398)(closes
 #18700)

---
 youtube_dl/extractor/radiocanada.py | 47 +++++++++++++++++---------
 youtube_dl/extractor/toutv.py       | 51 +++++++++--------------------
 2 files changed, 47 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 58e294892..dd95f99f2 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -4,12 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
     unified_strdate,
-    unsmuggle_url,
 )
 
 
@@ -58,23 +58,35 @@ class RadioCanadaIE(InfoExtractor):
         }
     ]
     _GEO_COUNTRIES = ['CA']
+    _access_token = None
+    _claims = None
 
-    def _call_api(self, path, video_id, app_code, query):
+    def _call_api(self, path, video_id=None, app_code=None, query=None):
+        if not query:
+            query = {}
         query.update({
-            'appCode': app_code,
-            'idMedia': video_id,
+            'client_key': '773aea60-0e80-41bb-9c7f-e6d7c3ad17fb',
             'output': 'json',
         })
-        return self._download_json(
-            'https://services.radio-canada.ca/media/' + path, video_id, headers={
-                'Authorization': 'Client-Key 773aea60-0e80-41bb-9c7f-e6d7c3ad17fb'
-            }, query=query)
+        if video_id:
+            query.update({
+                'appCode': app_code,
+                'idMedia': video_id,
+            })
+        if self._access_token:
+            query['access_token'] = self._access_token
+        try:
+            return self._download_json(
+                'https://services.radio-canada.ca/media/' + path, video_id, query=query)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 422):
+                data = self._parse_json(e.cause.read().decode(), None)
+                error = data.get('error_description') or data['errorMessage']['text']
+                raise ExtractorError(error, expected=True)
+            raise
 
-    def _real_extract(self, url):
-        url, smuggled_data = unsmuggle_url(url, {})
-        app_code, video_id = re.match(self._VALID_URL, url).groups()
-
-        metas = self._call_api('meta/v1/index.ashx', video_id, app_code, {})['Metas']
+    def _extract_info(self, app_code, video_id):
+        metas = self._call_api('meta/v1/index.ashx', video_id, app_code)['Metas']
 
         def get_meta(name):
             for meta in metas:
@@ -93,14 +105,16 @@ class RadioCanadaIE(InfoExtractor):
             'deviceType': 'ipad',
             'multibitrate': 'true',
         }
-        if smuggled_data:
-            query.update(smuggled_data)
+        if self._claims:
+            query['claims'] = self._claims
         v_data = self._call_api('validation/v2/', video_id, app_code, query)
         v_url = v_data.get('url')
         if not v_url:
             error = v_data['message']
             if error == "Le contenu sélectionné n'est pas disponible dans votre pays":
                 raise self.raise_geo_restricted(error, self._GEO_COUNTRIES)
+            if error == 'Le contenu sélectionné est disponible seulement en premium':
+                self.raise_login_required(error)
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, error), expected=True)
         formats = self._extract_m3u8_formats(v_url, video_id, 'mp4')
@@ -128,6 +142,9 @@ class RadioCanadaIE(InfoExtractor):
             'formats': formats,
         }
 
+    def _real_extract(self, url):
+        return self._extract_info(*re.match(self._VALID_URL, url).groups())
+
 
 class RadioCanadaAudioVideoIE(InfoExtractor):
     'radiocanada:audiovideo'
diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 2e7876cc5..f1ab91cf2 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -3,22 +3,19 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .radiocanada import RadioCanadaIE
 from ..utils import (
-    int_or_none,
-    js_to_json,
-    urlencode_postdata,
     extract_attributes,
-    smuggle_url,
+    int_or_none,
+    merge_dicts,
+    urlencode_postdata,
 )
 
 
-class TouTvIE(InfoExtractor):
+class TouTvIE(RadioCanadaIE):
     _NETRC_MACHINE = 'toutv'
     IE_NAME = 'tou.tv'
     _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/S[0-9]+[EC][0-9]+)?)'
-    _access_token = None
-    _claims = None
 
     _TESTS = [{
         'url': 'http://ici.tou.tv/garfield-tout-court/S2015E17',
@@ -46,18 +43,14 @@ class TouTvIE(InfoExtractor):
         email, password = self._get_login_info()
         if email is None:
             return
-        state = 'http://ici.tou.tv/'
-        webpage = self._download_webpage(state, None, 'Downloading homepage')
-        toutvlogin = self._parse_json(self._search_regex(
-            r'(?s)toutvlogin\s*=\s*({.+?});', webpage, 'toutvlogin'), None, js_to_json)
-        authorize_url = toutvlogin['host'] + '/auth/oauth/v2/authorize'
         login_webpage = self._download_webpage(
-            authorize_url, None, 'Downloading login page', query={
-                'client_id': toutvlogin['clientId'],
-                'redirect_uri': 'https://ici.tou.tv/login/loginCallback',
+            'https://services.radio-canada.ca/auth/oauth/v2/authorize',
+            None, 'Downloading login page', query={
+                'client_id': '4dd36440-09d5-4468-8923-b6d91174ad36',
+                'redirect_uri': 'https://ici.tou.tv/logincallback',
                 'response_type': 'token',
-                'scope': 'media-drmt openid profile email id.write media-validation.read.privileged',
-                'state': state,
+                'scope': 'id.write media-validation.read',
+                'state': '/',
             })
 
         def extract_form_url_and_data(wp, default_form_url, form_spec_re=''):
@@ -86,12 +79,7 @@ class TouTvIE(InfoExtractor):
         self._access_token = self._search_regex(
             r'access_token=([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             urlh.geturl(), 'access token')
-        self._claims = self._download_json(
-            'https://services.radio-canada.ca/media/validation/v2/getClaims',
-            None, 'Extracting Claims', query={
-                'token': self._access_token,
-                'access_token': self._access_token,
-            })['claims']
+        self._claims = self._call_api('validation/v2/getClaims')['claims']
 
     def _real_extract(self, url):
         path = self._match_id(url)
@@ -102,19 +90,10 @@ class TouTvIE(InfoExtractor):
             self.report_warning('This video is probably DRM protected.', path)
         video_id = metadata['IdMedia']
         details = metadata['Details']
-        title = details['OriginalTitle']
-        video_url = 'radiocanada:%s:%s' % (metadata.get('AppCode', 'toutv'), video_id)
-        if self._access_token and self._claims:
-            video_url = smuggle_url(video_url, {
-                'access_token': self._access_token,
-                'claims': self._claims,
-            })
 
-        return {
-            '_type': 'url_transparent',
-            'url': video_url,
+        return merge_dicts({
             'id': video_id,
-            'title': title,
+            'title': details.get('OriginalTitle'),
             'thumbnail': details.get('ImageUrl'),
             'duration': int_or_none(details.get('LengthInSeconds')),
-        }
+        }, self._extract_info(metadata.get('AppCode', 'toutv'), video_id))

From 241c5d7d384dcb01a62702274cfbead01f537145 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Feb 2019 19:38:10 +0100
Subject: [PATCH 401/510] [trutv] fix extraction(closes #17336)

---
 youtube_dl/extractor/trutv.py | 84 +++++++++++++++++++++++------------
 1 file changed, 56 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/trutv.py b/youtube_dl/extractor/trutv.py
index 3a5782525..ce892c8c5 100644
--- a/youtube_dl/extractor/trutv.py
+++ b/youtube_dl/extractor/trutv.py
@@ -4,44 +4,72 @@ from __future__ import unicode_literals
 import re
 
 from .turner import TurnerBaseIE
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class TruTVIE(TurnerBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?trutv\.com(?:(?P<path>/shows/[^/]+/videos/[^/?#]+?)\.html|/full-episodes/[^/]+/(?P<id>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?trutv\.com/(?:shows|full-episodes)/(?P<series_slug>[0-9A-Za-z-]+)/(?:videos/(?P<clip_slug>[0-9A-Za-z-]+)|(?P<id>\d+))'
     _TEST = {
-        'url': 'http://www.trutv.com/shows/10-things/videos/you-wont-believe-these-sports-bets.html',
-        'md5': '2cdc844f317579fed1a7251b087ff417',
+        'url': 'https://www.trutv.com/shows/the-carbonaro-effect/videos/sunlight-activated-flower.html',
         'info_dict': {
-            'id': '/shows/10-things/videos/you-wont-believe-these-sports-bets',
+            'id': 'f16c03beec1e84cd7d1a51f11d8fcc29124cc7f1',
             'ext': 'mp4',
-            'title': 'You Won\'t Believe These Sports Bets',
-            'description': 'Jamie Lee sits down with a bookie to discuss the bizarre world of illegal sports betting.',
-            'upload_date': '20130305',
-        }
+            'title': 'Sunlight-Activated Flower',
+            'description': "A customer is stunned when he sees Michael's sunlight-activated flower.",
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        path, video_id = re.match(self._VALID_URL, url).groups()
-        auth_required = False
-        if path:
-            data_src = 'http://www.trutv.com/video/cvp/v2/xml/content.xml?id=%s.xml' % path
+        series_slug, clip_slug, video_id = re.match(self._VALID_URL, url).groups()
+
+        if video_id:
+            path = 'episode'
+            display_id = video_id
         else:
-            webpage = self._download_webpage(url, video_id)
-            video_id = self._search_regex(
-                r"TTV\.TVE\.episodeId\s*=\s*'([^']+)';",
-                webpage, 'video id', default=video_id)
-            auth_required = self._search_regex(
-                r'TTV\.TVE\.authRequired\s*=\s*(true|false);',
-                webpage, 'auth required', default='false') == 'true'
-            data_src = 'http://www.trutv.com/tveverywhere/services/cvpXML.do?titleId=' + video_id
-        return self._extract_cvp_info(
-            data_src, path, {
-                'secure': {
-                    'media_src': 'http://androidhls-secure.cdn.turner.com/trutv/big',
-                    'tokenizer_src': 'http://www.trutv.com/tveverywhere/processors/services/token_ipadAdobe.do',
-                },
-            }, {
+            path = 'series/clip'
+            display_id = clip_slug
+
+        data = self._download_json(
+            'https://api.trutv.com/v2/web/%s/%s/%s' % (path, series_slug, display_id),
+            display_id)
+        video_data = data['episode'] if video_id else data['info']
+        media_id = video_data['mediaId']
+        title = video_data['title'].strip()
+
+        info = self._extract_ngtv_info(
+            media_id, {}, {
                 'url': url,
                 'site_name': 'truTV',
-                'auth_required': auth_required,
+                'auth_required': video_data.get('isAuthRequired'),
             })
+
+        thumbnails = []
+        for image in video_data.get('images', []):
+            image_url = image.get('srcUrl')
+            if not image_url:
+                continue
+            thumbnails.append({
+                'url': image_url,
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            })
+
+        info.update({
+            'id': media_id,
+            'display_id': display_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'thumbnails': thumbnails,
+            'timestamp': parse_iso8601(video_data.get('publicationDate')),
+            'series': video_data.get('showTitle'),
+            'season_number': int_or_none(video_data.get('seasonNum')),
+            'episode_number': int_or_none(video_data.get('episodeNum')),
+        })
+        return info

From f06a1cabe8b3831b5f2ff3bc27f5e7336c597e92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Feb 2019 23:57:58 +0700
Subject: [PATCH 402/510] [spankbang] Extend _VALID_URL

---
 youtube_dl/extractor/spankbang.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 67500b69c..e48cfff71 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class SpankBangIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|m|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
+    _VALID_URL = r'https?://(?:[^/]+\.)?spankbang\.com/(?P<id>[\da-z]+)/(?:video|play|embed)'
     _TESTS = [{
         'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
         'md5': '1cc433e1d6aa14bc376535b8679302f7',
@@ -41,13 +41,22 @@ class SpankBangIE(InfoExtractor):
         # 4k
         'url': 'https://spankbang.com/1vwqx/video/jade+kush+solo+4k',
         'only_matching': True,
+    }, {
+        'url': 'https://m.spankbang.com/3vvn/play/fantasy+solo/480p/',
+        'only_matching': True,
+    }, {
+        'url': 'https://m.spankbang.com/3vvn/play',
+        'only_matching': True,
+    }, {
+        'url': 'https://spankbang.com/2y3td/embed/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id, headers={
-            'Cookie': 'country=US'
-        })
+        webpage = self._download_webpage(
+            url.replace('/%s/embed' % video_id, '/%s/video' % video_id),
+            video_id, headers={'Cookie': 'country=US'})
 
         if re.search(r'<[^>]+\bid=["\']video_removed', webpage):
             raise ExtractorError(

From 49bd993fd9adbcf6b5c11a7ec11c2b4a552e49c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Feb 2019 00:09:50 +0700
Subject: [PATCH 403/510] [spankbang:playlist] Add extractor (closes #19145)

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/spankbang.py  | 33 +++++++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 693c16e49..d7685cd87 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1058,7 +1058,10 @@ from .southpark import (
     SouthParkEsIE,
     SouthParkNlIE
 )
-from .spankbang import SpankBangIE
+from .spankbang import (
+    SpankBangIE,
+    SpankBangPlaylistIE,
+)
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index e48cfff71..fbe6ef31a 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    orderedSet,
     parse_duration,
     parse_resolution,
     str_to_int,
@@ -12,7 +13,7 @@ from ..utils import (
 
 
 class SpankBangIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?spankbang\.com/(?P<id>[\da-z]+)/(?:video|play|embed)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?spankbang\.com/(?P<id>[\da-z]+)/(?:video|play|embed)\b'
     _TESTS = [{
         'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
         'md5': '1cc433e1d6aa14bc376535b8679302f7',
@@ -103,3 +104,33 @@ class SpankBangIE(InfoExtractor):
             'formats': formats,
             'age_limit': age_limit,
         }
+
+
+class SpankBangPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^/]+\.)?spankbang\.com/(?P<id>[\da-z]+)/playlist/[^/]+'
+    _TEST = {
+        'url': 'https://spankbang.com/ug0k/playlist/big+ass+titties',
+        'info_dict': {
+            'id': 'ug0k',
+            'title': 'Big Ass Titties',
+        },
+        'playlist_mincount': 50,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, playlist_id, headers={'Cookie': 'country=US; mobile=on'})
+
+        entries = [self.url_result(
+            'https://spankbang.com/%s/video' % video_id,
+            ie=SpankBangIE.ie_key(), video_id=video_id)
+            for video_id in orderedSet(re.findall(
+                r'<a[^>]+\bhref=["\']/?([\da-z]+)/play/', webpage))]
+
+        title = self._html_search_regex(
+            r'<h1>([^<]+)\s+playlist</h1>', webpage, 'playlist title',
+            fatal=False)
+
+        return self.playlist_result(entries, playlist_id, title)

From 22f5f5c6fcd2d7f0c9f1ff3019fe0b957b771f44 Mon Sep 17 00:00:00 2001
From: Ales Jirasek <schunkac@gmail.com>
Date: Wed, 10 Oct 2018 23:47:21 +0200
Subject: [PATCH 404/510] [malltv] Add extractor (closes #18058)

---
 test/test_InfoExtractor.py         |  2 ++
 youtube_dl/extractor/common.py     |  2 +-
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/malltv.py     | 58 ++++++++++++++++++++++++++++++
 youtube_dl/utils.py                |  2 +-
 5 files changed, 63 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/malltv.py

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 75fa0bbb7..f0aa8466b 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -61,6 +61,7 @@ class TestInfoExtractor(unittest.TestCase):
             <meta content='Foo' property=og:foobar>
             <meta name="og:test1" content='foo > < bar'/>
             <meta name="og:test2" content="foo >//< bar"/>
+            <meta property=og-test3 content='Ill-formatted opengraph'/>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
@@ -69,6 +70,7 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
         self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
         self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
+        self.assertEqual(ie._og_search_property('test3', html), 'Ill-formatted opengraph')
         self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
         self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
         self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c4ea2882f..c3b0586a0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1058,7 +1058,7 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
-        property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
+        property_re = (r'(?:name|property)=(?:\'og[:-]%(prop)s\'|"og[:-]%(prop)s"|\s*og[:-]%(prop)s\b)'
                        % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d7685cd87..f212b5116 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -619,6 +619,7 @@ from .mailru import (
     MailRuMusicSearchIE,
 )
 from .makertv import MakerTVIE
+from .malltv import MallTVIE
 from .mangomolo import (
     MangomoloVideoIE,
     MangomoloLiveIE,
diff --git a/youtube_dl/extractor/malltv.py b/youtube_dl/extractor/malltv.py
new file mode 100644
index 000000000..7e0876ecc
--- /dev/null
+++ b/youtube_dl/extractor/malltv.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import parse_duration, merge_dicts
+
+
+class MallTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mall\.tv/(?:.+/)?(?P<id>.+)(?:\?.*$|$)'
+    _TESTS = [
+        {
+            'url': 'https://www.mall.tv/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+            'md5': '9ced0de056534410837077e23bfba796',
+            'info_dict': {
+                'id': 't0zzt0',
+                'ext': 'mp4',
+                'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
+                'description': 'Pokud někdo hospodaří s penězmi daňových poplatníků, pak logicky chceme vědět, jak s nimi nakládá. Objem dotací pro neziskovky roste, ale opravdu jsou tyto organizace „pijavice", jak o nich hovoří And',
+                'upload_date': '20181007',
+                'timestamp': 1538870400
+                }
+        },
+        {
+            'url': 'https://www.mall.tv/kdo-to-plati/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+            'md5': '9ced0de056534410837077e23bfba796',
+            'only_matching': 1,
+            'info_dict': {
+                'id': 't0zzt0',
+                'ext': 'mp4',
+                'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
+                'description': 'Pokud někdo hospodaří s penězmi daňových poplatníků, pak logicky chceme vědět, jak s nimi nakládá. Objem dotací pro neziskovky roste, ale opravdu jsou tyto organizace „pijavice", jak o nich hovoří And',
+                'upload_date': '20181007',
+                'timestamp': 1538870400
+                }
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        src_id_regex = r'(?P<src><source src=([\"\'])?.+?/(?P<id>\w{6,}?)/index)(?P<after>\1?[^>]*?>)'
+        video_id = self._search_regex(src_id_regex, webpage, 'ID',
+                                           group='id')
+        info = self._search_json_ld(webpage, video_id, default={})
+        html = re.sub(src_id_regex, r'\g<src>.m3u8\g<after>', webpage)
+        media = self._parse_html5_media_entries(url, html, video_id)
+        thumbnail = info.get('thumbnail', self._og_search_thumbnail(webpage))
+        duration = parse_duration(info.get('duration'))
+        result = {
+            'id': video_id,
+            'title': info.get('title', self._og_search_title(webpage)),
+            'description': self._og_search_description(webpage)
+        }
+        result.update({'thumbnail': thumbnail})
+        result.update({'duration': duration})
+
+        return merge_dicts(media[0], info, result)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d0cb65814..f5a0bb4b0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -184,7 +184,7 @@ DATE_FORMATS_MONTH_FIRST.extend([
 ])
 
 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
-JSON_LD_RE = r'(?is)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
+JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
 
 
 def preferredencoding():

From 4de3cb883c61eeec56d8d271375a0624d481ad37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Feb 2019 00:43:10 +0700
Subject: [PATCH 405/510] [malltv] Fix issues and simplify (closes #17856)

---
 youtube_dl/extractor/malltv.py | 85 ++++++++++++++++------------------
 1 file changed, 40 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/malltv.py b/youtube_dl/extractor/malltv.py
index 7e0876ecc..e13c2e11a 100644
--- a/youtube_dl/extractor/malltv.py
+++ b/youtube_dl/extractor/malltv.py
@@ -2,57 +2,52 @@
 from __future__ import unicode_literals
 
 import re
+
 from .common import InfoExtractor
-from ..utils import parse_duration, merge_dicts
+from ..utils import merge_dicts
 
 
 class MallTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mall\.tv/(?:.+/)?(?P<id>.+)(?:\?.*$|$)'
-    _TESTS = [
-        {
-            'url': 'https://www.mall.tv/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
-            'md5': '9ced0de056534410837077e23bfba796',
-            'info_dict': {
-                'id': 't0zzt0',
-                'ext': 'mp4',
-                'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
-                'description': 'Pokud někdo hospodaří s penězmi daňových poplatníků, pak logicky chceme vědět, jak s nimi nakládá. Objem dotací pro neziskovky roste, ale opravdu jsou tyto organizace „pijavice", jak o nich hovoří And',
-                'upload_date': '20181007',
-                'timestamp': 1538870400
-                }
-        },
-        {
-            'url': 'https://www.mall.tv/kdo-to-plati/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
-            'md5': '9ced0de056534410837077e23bfba796',
-            'only_matching': 1,
-            'info_dict': {
-                'id': 't0zzt0',
-                'ext': 'mp4',
-                'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
-                'description': 'Pokud někdo hospodaří s penězmi daňových poplatníků, pak logicky chceme vědět, jak s nimi nakládá. Objem dotací pro neziskovky roste, ale opravdu jsou tyto organizace „pijavice", jak o nich hovoří And',
-                'upload_date': '20181007',
-                'timestamp': 1538870400
-                }
+    _VALID_URL = r'https?://(?:www\.)?mall\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.mall.tv/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+        'md5': '1c4a37f080e1f3023103a7b43458e518',
+        'info_dict': {
+            'id': 't0zzt0',
+            'display_id': '18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+            'ext': 'mp4',
+            'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
+            'description': 'md5:25fc0ec42a72ba602b602c683fa29deb',
+            'duration': 216,
+            'timestamp': 1538870400,
+            'upload_date': '20181007',
+            'view_count': int,
         }
-    ]
+    }, {
+        'url': 'https://www.mall.tv/kdo-to-plati/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        src_id_regex = r'(?P<src><source src=([\"\'])?.+?/(?P<id>\w{6,}?)/index)(?P<after>\1?[^>]*?>)'
-        video_id = self._search_regex(src_id_regex, webpage, 'ID',
-                                           group='id')
-        info = self._search_json_ld(webpage, video_id, default={})
-        html = re.sub(src_id_regex, r'\g<src>.m3u8\g<after>', webpage)
-        media = self._parse_html5_media_entries(url, html, video_id)
-        thumbnail = info.get('thumbnail', self._og_search_thumbnail(webpage))
-        duration = parse_duration(info.get('duration'))
-        result = {
-            'id': video_id,
-            'title': info.get('title', self._og_search_title(webpage)),
-            'description': self._og_search_description(webpage)
-        }
-        result.update({'thumbnail': thumbnail})
-        result.update({'duration': duration})
 
-        return merge_dicts(media[0], info, result)
+        webpage = self._download_webpage(
+            url, display_id, headers=self.geo_verification_headers())
+
+        SOURCE_RE = r'(<source[^>]+\bsrc=(?:(["\'])(?:(?!\2).)+|[^\s]+)/(?P<id>[\da-z]+)/index)\b'
+        video_id = self._search_regex(
+            SOURCE_RE, webpage, 'video id', group='id')
+
+        media = self._parse_html5_media_entries(
+            url, re.sub(SOURCE_RE, r'\1.m3u8', webpage), video_id,
+            m3u8_id='hls', m3u8_entry_protocol='m3u8_native')[0]
+
+        info = self._search_json_ld(webpage, video_id, default={})
+
+        return merge_dicts(media, info, {
+            'id': video_id,
+            'display_id': display_id,
+            'title': self._og_search_title(webpage, default=None) or display_id,
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
+        })

From 1211bb6dace4773f67bbf46b8944317679573a1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Feb 2019 01:08:48 +0700
Subject: [PATCH 406/510] [YoutubeDL] Improve _make_archive_id (closes #19149)

---
 youtube_dl/YoutubeDL.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c168415ce..bc9fc270c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -82,6 +82,7 @@ from .utils import (
     sanitize_url,
     sanitized_Request,
     std_headers,
+    str_or_none,
     subtitles_filename,
     UnavailableVideoError,
     url_basename,
@@ -2067,9 +2068,12 @@ class YoutubeDL(object):
         # and backwards compatibility with prior versions
         extractor = info_dict.get('extractor_key') or info_dict.get('ie_key')  # key in a playlist
         if extractor is None:
+            url = str_or_none(info_dict.get('url'))
+            if not url:
+                return
             # Try to find matching extractor for the URL and take its ie_key
             for ie in self._ies:
-                if ie.suitable(info_dict['url']):
+                if ie.suitable(url):
                     extractor = ie.ie_key()
                     break
             else:

From f1f5b47255a44e791a84b769c523127e0f047578 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Feb 2019 01:10:12 +0700
Subject: [PATCH 407/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 4872cd9fc..b0fbde43d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,32 @@
+version <unreleased>
+
+Core
+* [utils] Improve JSON-LD regular expression (#18058)
+* [YoutubeDL] Fallback to ie_key of matching extractor while making
+  download archive id when no explicit ie_key is provided (#19022)
+
+Extractors
++ [malltv] Add support for mall.tv (#18058, #17856)
++ [spankbang:playlist] Add support for playlists (#19145)
+* [spankbang] Extend URL regular expression
+* [trutv] Fix extraction (#17336)
+* [toutv] Fix authentication (#16398, #18700)
+* [pornhub] Fix tags and categories extraction (#13720, #19135)
+* [pornhd] Fix formats extraction
++ [pornhd] Extract like count (#19123, #19125)
+* [radiocanada] Switch to the new media requests (#19115)
++ [teachable] Add support for courses.workitdaily.com (#18871)
+- [vporn] Remove extractor (#16276)
++ [soundcloud:pagedplaylist] Add ie and title to entries (#19022, #19086)
++ [drtuber] Extract duration (#19078)
+* [soundcloud] Fix paged playlists extraction, add support for albums and update client id
+* [soundcloud] Update client id
+* [drtv] Improve preference (#19079)
++ [openload] Add support for openload.pw and oload.pw (#18930)
++ [openload] Add support for oload.info (#19073)
+* [crackle] Authorize media detail request (#16931)
+
+
 version 2019.01.30.1
 
 Core

From 04eacf54530e6a17129d3f1b90f759f9935f2b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Feb 2019 01:12:51 +0700
Subject: [PATCH 408/510] release 2019.02.08

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 423a08e4d..7128d998f 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.01.30.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.01.30.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.02.08*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.02.08**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.01.30.1
+[debug] youtube-dl version 2019.02.08
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index b0fbde43d..398528f76 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.02.08
 
 Core
 * [utils] Improve JSON-LD regular expression (#18058)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2918520c3..32fe6b647 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -476,6 +476,7 @@
  - **mailru:music**: Музыка@Mail.Ru
  - **mailru:music:search**: Музыка@Mail.Ru
  - **MakerTV**
+ - **MallTV**
  - **mangomolo:live**
  - **mangomolo:video**
  - **ManyVids**
@@ -827,6 +828,7 @@
  - **southpark.nl**
  - **southparkstudios.dk**
  - **SpankBang**
+ - **SpankBangPlaylist**
  - **Spankwire**
  - **Spiegel**
  - **Spiegel:Article**: Articles on spiegel.de
@@ -1057,7 +1059,6 @@
  - **Voot**
  - **VoxMedia**
  - **VoxMediaVolume**
- - **Vporn**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
  - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index be3bbdd73..4dc5a611e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.01.30.1'
+__version__ = '2019.02.08'

From 91effe22a091035bc5abace2fcf562a0db89090f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 8 Feb 2019 07:21:31 +0100
Subject: [PATCH 409/510] [linkedin:learning] extract chapter_number and
 chapter_id(closes #19162)

---
 youtube_dl/extractor/linkedin.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/linkedin.py b/youtube_dl/extractor/linkedin.py
index 259fc4c5e..5a86b0064 100644
--- a/youtube_dl/extractor/linkedin.py
+++ b/youtube_dl/extractor/linkedin.py
@@ -34,12 +34,15 @@ class LinkedInLearningBaseIE(InfoExtractor):
                 'Csrf-Token': self._get_cookies(api_url)['JSESSIONID'].value,
             }, query=query)['elements'][0]
 
-    def _get_video_id(self, urn, course_slug, video_slug):
+    def _get_urn_id(self, video_data):
+        urn = video_data.get('urn')
         if urn:
             mobj = re.search(r'urn:li:lyndaCourse:\d+,(\d+)', urn)
             if mobj:
                 return mobj.group(1)
-        return '%s/%s' % (course_slug, video_slug)
+
+    def _get_video_id(self, video_data, course_slug, video_slug):
+        return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug)
 
     def _real_initialize(self):
         email, password = self._get_login_info()
@@ -123,7 +126,7 @@ class LinkedInLearningIE(LinkedInLearningBaseIE):
         self._sort_formats(formats, ('width', 'height', 'source_preference', 'tbr', 'abr'))
 
         return {
-            'id': self._get_video_id(video_data.get('urn'), course_slug, video_slug),
+            'id': self._get_video_id(video_data, course_slug, video_slug),
             'title': title,
             'formats': formats,
             'thumbnail': video_data.get('defaultThumbnail'),
@@ -154,18 +157,21 @@ class LinkedInLearningCourseIE(LinkedInLearningBaseIE):
         course_data = self._call_api(course_slug, 'chapters,description,title')
 
         entries = []
-        for chapter in course_data.get('chapters', []):
+        for chapter_number, chapter in enumerate(course_data.get('chapters', []), 1):
             chapter_title = chapter.get('title')
+            chapter_id = self._get_urn_id(chapter)
             for video in chapter.get('videos', []):
                 video_slug = video.get('slug')
                 if not video_slug:
                     continue
                 entries.append({
                     '_type': 'url_transparent',
-                    'id': self._get_video_id(video.get('urn'), course_slug, video_slug),
+                    'id': self._get_video_id(video, course_slug, video_slug),
                     'title': video.get('title'),
                     'url': 'https://www.linkedin.com/learning/%s/%s' % (course_slug, video_slug),
                     'chapter': chapter_title,
+                    'chapter_number': chapter_number,
+                    'chapter_id': chapter_id,
                     'ie_key': LinkedInLearningIE.ie_key(),
                 })
 

From e9dee7f1b26e1c23b011a5ad5433d4debc6b48ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Feb 2019 23:49:37 +0700
Subject: [PATCH 410/510] [trunews] Add extractor (closes #19153)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/trunews.py    | 75 ++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/trunews.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f212b5116..3e1b63b4b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1171,6 +1171,7 @@ from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
+from .trunews import TruNewsIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
diff --git a/youtube_dl/extractor/trunews.py b/youtube_dl/extractor/trunews.py
new file mode 100644
index 000000000..b0c7caabf
--- /dev/null
+++ b/youtube_dl/extractor/trunews.py
@@ -0,0 +1,75 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    dict_get,
+    float_or_none,
+    int_or_none,
+    unified_timestamp,
+    update_url_query,
+    url_or_none,
+)
+
+
+class TruNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?trunews\.com/stream/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://www.trunews.com/stream/will-democrats-stage-a-circus-during-president-trump-s-state-of-the-union-speech',
+        'md5': 'a19c024c3906ff954fac9b96ce66bb08',
+        'info_dict': {
+            'id': '5c5a21e65d3c196e1c0020cc',
+            'display_id': 'will-democrats-stage-a-circus-during-president-trump-s-state-of-the-union-speech',
+            'ext': 'mp4',
+            'title': "Will Democrats Stage a Circus During President Trump's State of the Union Speech?",
+            'description': 'md5:c583b72147cc92cf21f56a31aff7a670',
+            'duration': 3685,
+            'timestamp': 1549411440,
+            'upload_date': '20190206',
+        },
+        'add_ie': ['Zype'],
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        video = self._download_json(
+            'https://api.zype.com/videos', display_id, query={
+                'app_key': 'PUVKp9WgGUb3-JUw6EqafLx8tFVP6VKZTWbUOR-HOm__g4fNDt1bCsm_LgYf_k9H',
+                'per_page': 1,
+                'active': 'true',
+                'friendly_title': display_id,
+            })['response'][0]
+
+        zype_id = video['_id']
+
+        thumbnails = []
+        thumbnails_list = video.get('thumbnails')
+        if isinstance(thumbnails_list, list):
+            for thumbnail in thumbnails_list:
+                if not isinstance(thumbnail, dict):
+                    continue
+                thumbnail_url = url_or_none(thumbnail.get('url'))
+                if not thumbnail_url:
+                    continue
+                thumbnails.append({
+                    'url': thumbnail_url,
+                    'width': int_or_none(thumbnail.get('width')),
+                    'height': int_or_none(thumbnail.get('height')),
+                })
+
+        return {
+            '_type': 'url_transparent',
+            'url': update_url_query(
+                'https://player.zype.com/embed/%s.js' % zype_id,
+                {'api_key': 'X5XnahkjCwJrT_l5zUqypnaLEObotyvtUKJWWlONxDoHVjP8vqxlArLV8llxMbyt'}),
+            'ie_key': 'Zype',
+            'id': zype_id,
+            'display_id': display_id,
+            'title': video.get('title'),
+            'description': dict_get(video, ('description', 'ott_description', 'short_description')),
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': unified_timestamp(video.get('published_at')),
+            'average_rating': float_or_none(video.get('rating')),
+            'view_count': int_or_none(video.get('request_count')),
+            'thumbnails': thumbnails,
+        }

From f516f44094d2244a805d8d0ac4d809fc6cd16782 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Feb 2019 23:44:08 +0700
Subject: [PATCH 411/510] [soundcloud] Extract more metadata

---
 youtube_dl/extractor/soundcloud.py | 65 +++++++++++++++++++++++++-----
 1 file changed, 56 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 5536e7851..15da3496e 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -16,7 +16,8 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
-    unified_strdate,
+    try_get,
+    unified_timestamp,
     update_url_query,
     url_or_none,
 )
@@ -51,12 +52,17 @@ class SoundcloudIE(InfoExtractor):
             'info_dict': {
                 'id': '62986583',
                 'ext': 'mp3',
-                'upload_date': '20121011',
+                'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
                 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
                 'uploader': 'E.T. ExTerrestrial Music',
-                'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
+                'timestamp': 1349920598,
+                'upload_date': '20121011',
                 'duration': 143,
                 'license': 'all-rights-reserved',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             }
         },
         # not streamable song
@@ -68,9 +74,14 @@ class SoundcloudIE(InfoExtractor):
                 'title': 'Goldrushed',
                 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
                 'uploader': 'The Royal Concept',
+                'timestamp': 1337635207,
                 'upload_date': '20120521',
-                'duration': 227,
+                'duration': 30,
                 'license': 'all-rights-reserved',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
             'params': {
                 # rtmp
@@ -85,11 +96,16 @@ class SoundcloudIE(InfoExtractor):
                 'id': '123998367',
                 'ext': 'mp3',
                 'title': 'Youtube - Dl Test Video \'\' Ä↭',
-                'uploader': 'jaimeMF',
                 'description': 'test chars:  \"\'/\\ä↭',
+                'uploader': 'jaimeMF',
+                'timestamp': 1386604920,
                 'upload_date': '20131209',
                 'duration': 9,
                 'license': 'all-rights-reserved',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
         },
         # private link (alt format)
@@ -100,11 +116,16 @@ class SoundcloudIE(InfoExtractor):
                 'id': '123998367',
                 'ext': 'mp3',
                 'title': 'Youtube - Dl Test Video \'\' Ä↭',
-                'uploader': 'jaimeMF',
                 'description': 'test chars:  \"\'/\\ä↭',
+                'uploader': 'jaimeMF',
+                'timestamp': 1386604920,
                 'upload_date': '20131209',
                 'duration': 9,
                 'license': 'all-rights-reserved',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
         },
         # downloadable song
@@ -117,9 +138,14 @@ class SoundcloudIE(InfoExtractor):
                 'title': 'Bus Brakes',
                 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
                 'uploader': 'oddsamples',
+                'timestamp': 1389232924,
                 'upload_date': '20140109',
                 'duration': 17,
                 'license': 'cc-by-sa',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
         },
         # private link, downloadable format
@@ -132,9 +158,14 @@ class SoundcloudIE(InfoExtractor):
                 'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]',
                 'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366',
                 'uploader': 'Ori Uplift Music',
+                'timestamp': 1504206263,
                 'upload_date': '20170831',
                 'duration': 7449,
                 'license': 'all-rights-reserved',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
         },
         # no album art, use avatar pic for thumbnail
@@ -147,10 +178,15 @@ class SoundcloudIE(InfoExtractor):
                 'title': 'Sideways (Prod. Mad Real)',
                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
                 'uploader': 'garyvee',
+                'timestamp': 1488152409,
                 'upload_date': '20170226',
                 'duration': 207,
                 'thumbnail': r're:https?://.*\.jpg',
                 'license': 'all-rights-reserved',
+                'view_count': int,
+                'like_count': int,
+                'comment_count': int,
+                'repost_count': int,
             },
             'params': {
                 'skip_download': True,
@@ -176,22 +212,33 @@ class SoundcloudIE(InfoExtractor):
 
     def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
         track_id = compat_str(info['id'])
+        title = info['title']
         name = full_title or track_id
         if quiet:
             self.report_extraction(name)
         thumbnail = info.get('artwork_url') or info.get('user', {}).get('avatar_url')
         if isinstance(thumbnail, compat_str):
             thumbnail = thumbnail.replace('-large', '-t500x500')
+        username = try_get(info, lambda x: x['user']['username'], compat_str)
+
+        def extract_count(key):
+            return int_or_none(info.get('%s_count' % key))
+
         result = {
             'id': track_id,
-            'uploader': info.get('user', {}).get('username'),
-            'upload_date': unified_strdate(info.get('created_at')),
-            'title': info['title'],
+            'uploader': username,
+            'timestamp': unified_timestamp(info.get('created_at')),
+            'title': title,
             'description': info.get('description'),
             'thumbnail': thumbnail,
             'duration': int_or_none(info.get('duration'), 1000),
             'webpage_url': info.get('permalink_url'),
             'license': info.get('license'),
+            'view_count': extract_count('playback'),
+            'like_count': extract_count('favoritings'),
+            'comment_count': extract_count('comment'),
+            'repost_count': extract_count('reposts'),
+            'genre': info.get('genre'),
         }
         formats = []
         query = {'client_id': self._CLIENT_ID}

From 4c0e0dc9dc13d53a334f75f7d7b9073f79a2dfc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Feb 2019 00:49:51 +0700
Subject: [PATCH 412/510] [rutube:embed] Fix extraction and add support private
 videos (closes #19163)

---
 youtube_dl/extractor/rutube.py | 115 +++++++++++++++++++++------------
 1 file changed, 72 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 10ac8ed1f..8f54d5675 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -21,7 +21,17 @@ from ..utils import (
 
 
 class RutubeBaseIE(InfoExtractor):
-    def _extract_video(self, video, video_id=None, require_title=True):
+    def _download_api_info(self, video_id, query=None):
+        if not query:
+            query = {}
+        query['format'] = 'json'
+        return self._download_json(
+            'http://rutube.ru/api/video/%s/' % video_id,
+            video_id, 'Downloading video JSON',
+            'Unable to download video JSON', query=query)
+
+    @staticmethod
+    def _extract_info(video, video_id=None, require_title=True):
         title = video['title'] if require_title else video.get('title')
 
         age_limit = video.get('is_adult')
@@ -32,7 +42,7 @@ class RutubeBaseIE(InfoExtractor):
         category = try_get(video, lambda x: x['category']['name'])
 
         return {
-            'id': video.get('id') or video_id,
+            'id': video.get('id') or video_id if video_id else video['id'],
             'title': title,
             'description': video.get('description'),
             'thumbnail': video.get('thumbnail_url'),
@@ -47,6 +57,42 @@ class RutubeBaseIE(InfoExtractor):
             'is_live': bool_or_none(video.get('is_livestream')),
         }
 
+    def _download_and_extract_info(self, video_id, query=None):
+        return self._extract_info(
+            self._download_api_info(video_id, query=query), video_id)
+
+    def _download_api_options(self, video_id, query=None):
+        if not query:
+            query = {}
+        query['format'] = 'json'
+        return self._download_json(
+            'http://rutube.ru/api/play/options/%s/' % video_id,
+            video_id, 'Downloading options JSON',
+            'Unable to download options JSON',
+            headers=self.geo_verification_headers(), query=query)
+
+    def _extract_formats(self, options, video_id):
+        formats = []
+        for format_id, format_url in options['video_balancer'].items():
+            ext = determine_ext(format_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id, fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def _download_and_extract_formats(self, video_id, query=None):
+        return self._extract_formats(
+            self._download_api_options(video_id, query=query), video_id)
+
 
 class RutubeIE(RutubeBaseIE):
     IE_NAME = 'rutube'
@@ -55,13 +101,13 @@ class RutubeIE(RutubeBaseIE):
 
     _TESTS = [{
         'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
-        'md5': '79938ade01294ef7e27574890d0d3769',
+        'md5': '1d24f180fac7a02f3900712e5a5764d6',
         'info_dict': {
             'id': '3eac3b4561676c17df9132a9a1e62e3e',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Раненный кенгуру забежал в аптеку',
             'description': 'http://www.ntdtv.ru ',
-            'duration': 80,
+            'duration': 81,
             'uploader': 'NTDRussian',
             'uploader_id': '29790',
             'timestamp': 1381943602,
@@ -94,39 +140,12 @@ class RutubeIE(RutubeBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        video = self._download_json(
-            'http://rutube.ru/api/video/%s/?format=json' % video_id,
-            video_id, 'Downloading video JSON')
-
-        info = self._extract_video(video, video_id)
-
-        options = self._download_json(
-            'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
-            video_id, 'Downloading options JSON',
-            headers=self.geo_verification_headers())
-
-        formats = []
-        for format_id, format_url in options['video_balancer'].items():
-            ext = determine_ext(format_url)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    format_url, video_id, f4m_id=format_id, fatal=False))
-            else:
-                formats.append({
-                    'url': format_url,
-                    'format_id': format_id,
-                })
-        self._sort_formats(formats)
-
-        info['formats'] = formats
+        info = self._download_and_extract_info(video_id)
+        info['formats'] = self._download_and_extract_formats(video_id)
         return info
 
 
-class RutubeEmbedIE(InfoExtractor):
+class RutubeEmbedIE(RutubeBaseIE):
     IE_NAME = 'rutube:embed'
     IE_DESC = 'Rutube embedded videos'
     _VALID_URL = r'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
@@ -135,7 +154,7 @@ class RutubeEmbedIE(InfoExtractor):
         'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
         'info_dict': {
             'id': 'a10e53b86e8f349080f718582ce4c661',
-            'ext': 'flv',
+            'ext': 'mp4',
             'timestamp': 1387830582,
             'upload_date': '20131223',
             'uploader_id': '297833',
@@ -149,16 +168,26 @@ class RutubeEmbedIE(InfoExtractor):
     }, {
         'url': 'http://rutube.ru/play/embed/8083783',
         'only_matching': True,
+    }, {
+        # private video
+        'url': 'https://rutube.ru/play/embed/10631925?p=IbAigKqWd1do4mjaM5XLIQ',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         embed_id = self._match_id(url)
-        webpage = self._download_webpage(url, embed_id)
-
-        canonical_url = self._html_search_regex(
-            r'<link\s+rel="canonical"\s+href="([^"]+?)"', webpage,
-            'Canonical URL')
-        return self.url_result(canonical_url, RutubeIE.ie_key())
+        # Query may contain private videos token and should be passed to API
+        # requests (see #19163)
+        query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        options = self._download_api_options(embed_id, query)
+        video_id = options['effective_video']
+        formats = self._extract_formats(options, video_id)
+        info = self._download_and_extract_info(video_id, query)
+        info.update({
+            'extractor_key': 'Rutube',
+            'formats': formats,
+        })
+        return info
 
 
 class RutubePlaylistBaseIE(RutubeBaseIE):
@@ -181,7 +210,7 @@ class RutubePlaylistBaseIE(RutubeBaseIE):
                 video_url = url_or_none(result.get('video_url'))
                 if not video_url:
                     continue
-                entry = self._extract_video(result, require_title=False)
+                entry = self._extract_info(result, require_title=False)
                 entry.update({
                     '_type': 'url',
                     'url': video_url,

From d777f3e81c5f9f6c7be39586703f1f076a2025a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Feb 2019 04:39:23 +0700
Subject: [PATCH 413/510] [tvplayhome] Fix episode metadata extraction (closes
 #19190)

---
 youtube_dl/extractor/tvplay.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 8f1ff3b76..7c07b26bc 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -537,8 +537,9 @@ class TVPlayHomeIE(InfoExtractor):
             r'(\d+)(?:[.\s]+sezona|\s+HOOAEG)', season or '', 'season number',
             default=None))
         episode = self._search_regex(
-            r'(["\'])(?P<value>(?:(?!\1).)+)\1', webpage, 'episode',
-            default=None, group='value')
+            (r'\bepisode\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
+             r'data-subtitle\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
+            'episode', default=None, group='value')
         episode_number = int_or_none(self._search_regex(
             r'(?:S[eē]rija|Osa)\s+(\d+)', episode or '', 'episode number',
             default=None))

From 7d8b89163c43dfec27dab5250183e52fce838389 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Feb 2019 04:41:28 +0700
Subject: [PATCH 414/510] [tvplayhome] Fix video id extraction (closes #19190)

---
 youtube_dl/extractor/tvplay.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 7c07b26bc..d82d48f94 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -493,10 +493,9 @@ class TVPlayHomeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_id = self._search_regex(
-            r'data-asset-id\s*=\s*["\'](\d{5,7})\b', webpage, 'video id',
-            default=None)
+            r'data-asset-id\s*=\s*["\'](\d{5,})\b', webpage, 'video id')
 
-        if video_id:
+        if len(video_id) < 8:
             return self.url_result(
                 'mtg:%s' % video_id, ie=TVPlayIE.ie_key(), video_id=video_id)
 

From 985637cbbfc1a79091eb2f5ca4afec84f6616c75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Feb 2019 00:13:50 +0700
Subject: [PATCH 415/510] [twitch] Add new source format detection approach
 (closes #19193)

---
 youtube_dl/extractor/twitch.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 401615683..8c87f6dd3 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -136,7 +136,12 @@ class TwitchBaseIE(InfoExtractor):
             source = next(f for f in formats if f['format_id'] == 'Source')
             source['preference'] = 10
         except StopIteration:
-            pass  # No Source stream present
+            for f in formats:
+                if '/chunked/' in f['url']:
+                    f.update({
+                        'source_preference': 10,
+                        'format_note': 'Source',
+                    })
         self._sort_formats(formats)
 
 

From 6f5c1807f43b9dfd17fdc6932ae7ecb6c77fb1a0 Mon Sep 17 00:00:00 2001
From: bitraid <bitraid@protonmail.ch>
Date: Tue, 12 Feb 2019 19:02:29 +0200
Subject: [PATCH 416/510] [imgur] Use video id as title fallback (closes
 #18590)

---
 youtube_dl/extractor/imgur.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 0eb54db3f..a5ba03efa 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -27,6 +27,10 @@ class ImgurIE(InfoExtractor):
     }, {
         'url': 'https://i.imgur.com/crGpqCV.mp4',
         'only_matching': True,
+    }, {
+        # no title
+        'url': 'https://i.imgur.com/jxBXAMC.gifv',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -87,7 +91,7 @@ class ImgurIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': self._og_search_title(webpage),
+            'title': self._og_search_title(webpage, default=video_id),
         }
 
 

From 7bee705d8f110f09d8e72b1c863ff197ccc1d4f1 Mon Sep 17 00:00:00 2001
From: yonaikerlol <lawlietrs7@gmail.com>
Date: Thu, 14 Feb 2019 11:28:16 -0400
Subject: [PATCH 417/510] [openload] Add support for oload.live

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index a2ae25272..c1dcbb7eb 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -249,7 +249,7 @@ class OpenloadIE(InfoExtractor):
                             (?:www\.)?
                             (?:
                                 openload\.(?:co|io|link|pw)|
-                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw)
+                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live)
                             )
                         )/
                         (?:f|embed)/
@@ -346,6 +346,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.pw/f/WyKgK8s94N0',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.live/f/-Z58UZ-GR4M',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From 794c1b6e02591b04da931fa59745bc47bfae7492 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Feb 2019 23:40:46 +0700
Subject: [PATCH 418/510] [vshare] Pass Referer to download request (closes
 #19205, closes #19221)

---
 youtube_dl/extractor/vshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index e4ec77889..c631ac1fa 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -48,7 +48,7 @@ class VShareIE(InfoExtractor):
 
         webpage = self._download_webpage(
             'https://vshare.io/v/%s/width-650/height-430/1' % video_id,
-            video_id)
+            video_id, headers={'Referer': url})
 
         title = self._html_search_regex(
             r'<title>([^<]+)</title>', webpage, 'title')

From 2b2da3ba10cc325d00b665aae87f0fa8508bccdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Feb 2019 23:56:29 +0700
Subject: [PATCH 419/510] [rai] Relax _VALID_URL (closes #19232)

---
 youtube_dl/extractor/rai.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 548a6553b..149153b8f 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -288,7 +288,7 @@ class RaiPlayPlaylistIE(InfoExtractor):
 
 
 class RaiIE(RaiBaseIE):
-    _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
+    _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{
         # var uniquename = "ContentItem-..."
         # data-id="ContentItem-..."
@@ -375,6 +375,9 @@ class RaiIE(RaiBaseIE):
         # Direct MMS URL
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.rainews.it/tgr/marche/notiziari/video/2019/02/ContentItem-6ba945a2-889c-4a80-bdeb-8489c70a8db9.html',
+        'only_matching': True,
     }]
 
     def _extract_from_content_id(self, content_id, url):

From ba2e3730d125eab952eded3bb7749d479a2262d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Feb 2019 22:45:53 +0700
Subject: [PATCH 420/510] [noovo] Fix extraction (closes #19230)

---
 youtube_dl/extractor/noovo.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
index 974de3c3e..b40770d07 100644
--- a/youtube_dl/extractor/noovo.py
+++ b/youtube_dl/extractor/noovo.py
@@ -57,7 +57,8 @@ class NoovoIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        bc_url = BrightcoveNewIE._extract_url(self, webpage)
+        brightcove_id = self._search_regex(
+            r'data-video-id=["\'](\d+)', webpage, 'brightcove id')
 
         data = self._parse_json(
             self._search_regex(
@@ -89,7 +90,10 @@ class NoovoIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             'ie_key': BrightcoveNewIE.ie_key(),
-            'url': smuggle_url(bc_url, {'geo_countries': ['CA']}),
+            'url': smuggle_url(
+                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+                {'geo_countries': ['CA']}),
+            'id': brightcove_id,
             'title': title,
             'description': description,
             'series': series,

From ae65c93a26f2b3cf806477a3ee891aa461b5c6b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 00:58:13 +0700
Subject: [PATCH 421/510] [udemy] Update User-Agent and detect captcha (closes
 #14713, closes #15839, closes #18126)

---
 youtube_dl/extractor/udemy.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 105826e9b..89a7f6ade 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -123,10 +123,22 @@ class UdemyIE(InfoExtractor):
 
     def _download_webpage_handle(self, *args, **kwargs):
         headers = kwargs.get('headers', {}).copy()
-        headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4'
+        headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
         kwargs['headers'] = headers
-        return super(UdemyIE, self)._download_webpage_handle(
+        ret = super(UdemyIE, self)._download_webpage_handle(
             *args, **compat_kwargs(kwargs))
+        if not ret:
+            return ret
+        webpage, _ = ret
+        if any(p in webpage for p in (
+                '>Please verify you are a human',
+                'Access to this page has been denied because we believe you are using automation tools to browse the website',
+                '"_pxCaptcha"')):
+            raise ExtractorError(
+                'Udemy asks you to solve a CAPTCHA. Login with browser, '
+                'solve CAPTCHA, then export cookies and pass cookie file to '
+                'youtube-dl with --cookies.', expected=True)
+        return ret
 
     def _download_json(self, url_or_request, *args, **kwargs):
         headers = {

From d7d513891b7e63337218c5cb0bf743c8f7044381 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 01:05:01 +0700
Subject: [PATCH 422/510] [udemy] Extend _VALID_URLs (closes #14330, closes
 #15883)

---
 youtube_dl/extractor/udemy.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 89a7f6ade..ae8de9897 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -29,7 +29,7 @@ class UdemyIE(InfoExtractor):
     IE_NAME = 'udemy'
     _VALID_URL = r'''(?x)
                     https?://
-                        www\.udemy\.com/
+                        (?:[^/]+\.)?udemy\.com/
                         (?:
                             [^#]+\#/lecture/|
                             lecture/view/?\?lectureId=|
@@ -64,6 +64,9 @@ class UdemyIE(InfoExtractor):
         # only outputs rendition
         'url': 'https://www.udemy.com/how-you-can-help-your-local-community-5-amazing-examples/learn/v4/t/lecture/3225750?start=0',
         'only_matching': True,
+    }, {
+        'url': 'https://wipro.udemy.com/java-tutorial/#/lecture/172757',
+        'only_matching': True,
     }]
 
     def _extract_course_info(self, webpage, video_id):
@@ -415,8 +418,14 @@ class UdemyIE(InfoExtractor):
 
 class UdemyCourseIE(UdemyIE):
     IE_NAME = 'udemy:course'
-    _VALID_URL = r'https?://(?:www\.)?udemy\.com/(?P<id>[^/?#&]+)'
-    _TESTS = []
+    _VALID_URL = r'https?://(?:[^/]+\.)?udemy\.com/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.udemy.com/java-tutorial/',
+        'only_matching': True,
+    }, {
+        'url': 'https://wipro.udemy.com/java-tutorial/',
+        'only_matching': True,
+    }]
 
     @classmethod
     def suitable(cls, url):

From c9a0ea6e51eff28b9a383a47215870fd5875fc3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 05:00:16 +0700
Subject: [PATCH 423/510] [bilibili] Update keys (closes #19233)

---
 youtube_dl/extractor/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 4d6b051fe..3746671d3 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -93,8 +93,8 @@ class BiliBiliIE(InfoExtractor):
         }]
     }]
 
-    _APP_KEY = '84956560bc028eb7'
-    _BILIBILI_KEY = '94aba54af9065f71de72f5508f1cd42e'
+    _APP_KEY = 'iVGUTjsxvpLeuDCf'
+    _BILIBILI_KEY = 'aHRmhWMLkdeMuILqORnYZocwMBpMEOdt'
 
     def _report_error(self, result):
         if 'message' in result:

From 659e93fcf5c0480ac461cda412335cecf6a5595f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 07:12:10 +0700
Subject: [PATCH 424/510] [linuxacademy] Add extractor (closes #12207)

---
 youtube_dl/extractor/extractors.py   |   1 +
 youtube_dl/extractor/linuxacademy.py | 174 +++++++++++++++++++++++++++
 2 files changed, 175 insertions(+)
 create mode 100644 youtube_dl/extractor/linuxacademy.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3e1b63b4b..c70452dcd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -593,6 +593,7 @@ from .linkedin import (
     LinkedInLearningIE,
     LinkedInLearningCourseIE,
 )
+from .linuxacademy import LinuxAcademyIE
 from .litv import LiTVIE
 from .liveleak import (
     LiveLeakIE,
diff --git a/youtube_dl/extractor/linuxacademy.py b/youtube_dl/extractor/linuxacademy.py
new file mode 100644
index 000000000..a78c6556e
--- /dev/null
+++ b/youtube_dl/extractor/linuxacademy.py
@@ -0,0 +1,174 @@
+from __future__ import unicode_literals
+
+import json
+import random
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_b64decode,
+    compat_HTTPError,
+    compat_str,
+)
+from ..utils import (
+    ExtractorError,
+    orderedSet,
+    unescapeHTML,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class LinuxAcademyIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?linuxacademy\.com/cp/
+                        (?:
+                            courses/lesson/course/(?P<chapter_id>\d+)/lesson/(?P<lesson_id>\d+)|
+                            modules/view/id/(?P<course_id>\d+)
+                        )
+                    '''
+    _TESTS = [{
+        'url': 'https://linuxacademy.com/cp/courses/lesson/course/1498/lesson/2/module/154',
+        'info_dict': {
+            'id': '1498-2',
+            'ext': 'mp4',
+            'title': "Introduction to the Practitioner's Brief",
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'skip': 'Requires Linux Academy account credentials',
+    }, {
+        'url': 'https://linuxacademy.com/cp/courses/lesson/course/1498/lesson/2',
+        'only_matching': True,
+    }, {
+        'url': 'https://linuxacademy.com/cp/modules/view/id/154',
+        'info_dict': {
+            'id': '154',
+            'title': 'AWS Certified Cloud Practitioner',
+            'description': 'md5:039db7e60e4aac9cf43630e0a75fa834',
+        },
+        'playlist_count': 41,
+        'skip': 'Requires Linux Academy account credentials',
+    }]
+
+    _AUTHORIZE_URL = 'https://login.linuxacademy.com/authorize'
+    _ORIGIN_URL = 'https://linuxacademy.com'
+    _CLIENT_ID = 'KaWxNn1C2Gc7n83W9OFeXltd8Utb5vvx'
+    _NETRC_MACHINE = 'linuxacademy'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        def random_string():
+            return ''.join([
+                random.choice('0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~')
+                for _ in range(32)])
+
+        webpage, urlh = self._download_webpage_handle(
+            self._AUTHORIZE_URL, None, 'Downloading authorize page', query={
+                'client_id': self._CLIENT_ID,
+                'response_type': 'token id_token',
+                'redirect_uri': self._ORIGIN_URL,
+                'scope': 'openid email user_impersonation profile',
+                'audience': self._ORIGIN_URL,
+                'state': random_string(),
+                'nonce': random_string(),
+            })
+
+        login_data = self._parse_json(
+            self._search_regex(
+                r'atob\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+                'login info', group='value'), None,
+            transform_source=lambda x: compat_b64decode(x).decode('utf-8')
+        )['extraParams']
+
+        login_data.update({
+            'client_id': self._CLIENT_ID,
+            'redirect_uri': self._ORIGIN_URL,
+            'tenant': 'lacausers',
+            'connection': 'Username-Password-Authentication',
+            'username': username,
+            'password': password,
+            'sso': 'true',
+        })
+
+        login_state_url = compat_str(urlh.geturl())
+
+        try:
+            login_page = self._download_webpage(
+                'https://login.linuxacademy.com/usernamepassword/login', None,
+                'Downloading login page', data=json.dumps(login_data).encode(),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Origin': 'https://login.linuxacademy.com',
+                    'Referer': login_state_url,
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                error = self._parse_json(e.cause.read(), None)
+                message = error.get('description') or error['code']
+                raise ExtractorError(
+                    '%s said: %s' % (self.IE_NAME, message), expected=True)
+            raise
+
+        callback_page, urlh = self._download_webpage_handle(
+            'https://login.linuxacademy.com/login/callback', None,
+            'Downloading callback page',
+            data=urlencode_postdata(self._hidden_inputs(login_page)),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Origin': 'https://login.linuxacademy.com',
+                'Referer': login_state_url,
+            })
+
+        access_token = self._search_regex(
+            r'access_token=([^=&]+)', compat_str(urlh.geturl()),
+            'access token')
+
+        self._download_webpage(
+            'https://linuxacademy.com/cp/login/tokenValidateLogin/token/%s'
+            % access_token, None, 'Downloading token validation page')
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        chapter_id, lecture_id, course_id = mobj.group('chapter_id', 'lesson_id', 'course_id')
+        item_id = course_id if course_id else '%s-%s' % (chapter_id, lecture_id)
+
+        webpage = self._download_webpage(url, item_id)
+
+        # course path
+        if course_id:
+            entries = [
+                self.url_result(
+                    urljoin(url, lesson_url), ie=LinuxAcademyIE.ie_key())
+                for lesson_url in orderedSet(re.findall(
+                    r'<a[^>]+\bhref=["\'](/cp/courses/lesson/course/\d+/lesson/\d+/module/\d+)',
+                    webpage))]
+            title = unescapeHTML(self._html_search_regex(
+                (r'class=["\']course-title["\'][^>]*>(?P<value>[^<]+)',
+                 r'var\s+title\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1'),
+                webpage, 'title', default=None, group='value'))
+            description = unescapeHTML(self._html_search_regex(
+                r'var\s+description\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
+                webpage, 'description', default=None, group='value'))
+            return self.playlist_result(entries, course_id, title, description)
+
+        # single video path
+        info = self._extract_jwplayer_data(
+            webpage, item_id, require_title=False, m3u8_id='hls',)
+        title = self._search_regex(
+            (r'>Lecture\s*:\s*(?P<value>[^<]+)',
+             r'lessonName\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
+            'title', group='value')
+        info.update({
+            'id': item_id,
+            'title': title,
+        })
+        return info

From 3c9647372e78134777d201e157a5ef42345c9da2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 13:38:21 +0700
Subject: [PATCH 425/510] [tvp] Fix description extraction, make thumbnail
 optional and fix tests

---
 youtube_dl/extractor/tvp.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 3954f0b93..f9bf600b0 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -19,12 +19,12 @@ class TVPIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://vod.tvp.pl/video/czas-honoru,i-seria-odc-13,194536',
-        'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
+        'md5': 'a21eb0aa862f25414430f15fdfb9e76c',
         'info_dict': {
             'id': '194536',
             'ext': 'mp4',
-            'title': 'Czas honoru, I seria – odc. 13',
-            'description': 'md5:381afa5bca72655fe94b05cfe82bf53d',
+            'title': 'Czas honoru, odc. 13 – Władek',
+            'description': 'md5:437f48b93558370b031740546b696e24',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
@@ -45,6 +45,7 @@ class TVPIE(InfoExtractor):
             'title': 'Wiadomości, 28.09.2017, 19:30',
             'description': 'Wydanie główne codziennego serwisu informacyjnego.'
         },
+        'skip': 'HTTP Error 404: Not Found',
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
         'only_matching': True,
@@ -75,8 +76,10 @@ class TVPIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             'url': 'tvp:' + video_id,
-            'description': self._og_search_description(webpage, default=None),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(
+                webpage, default=None) or self._html_search_meta(
+                'description', webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'ie_key': 'TVPEmbed',
         }
 
@@ -87,6 +90,14 @@ class TVPEmbedIE(InfoExtractor):
     _VALID_URL = r'(?:tvp:|https?://[^/]+\.tvp\.(?:pl|info)/sess/tvplayer\.php\?.*?object_id=)(?P<id>\d+)'
 
     _TESTS = [{
+        'url': 'tvp:194536',
+        'md5': 'a21eb0aa862f25414430f15fdfb9e76c',
+        'info_dict': {
+            'id': '194536',
+            'ext': 'mp4',
+            'title': 'Czas honoru, odc. 13 – Władek',
+        },
+    }, {
         'url': 'http://www.tvp.pl/sess/tvplayer.php?object_id=22670268',
         'md5': '8c9cd59d16edabf39331f93bf8a766c7',
         'info_dict': {

From 34568dc2967d227630ed9d7150deaa62a689b937 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 13:39:00 +0700
Subject: [PATCH 426/510] [tvp] Detect unavailable videos

---
 youtube_dl/extractor/tvp.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index f9bf600b0..536b580fc 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -98,6 +98,7 @@ class TVPEmbedIE(InfoExtractor):
             'title': 'Czas honoru, odc. 13 – Władek',
         },
     }, {
+        # not available
         'url': 'http://www.tvp.pl/sess/tvplayer.php?object_id=22670268',
         'md5': '8c9cd59d16edabf39331f93bf8a766c7',
         'info_dict': {
@@ -105,6 +106,7 @@ class TVPEmbedIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Panorama, 07.12.2015, 15:40',
         },
+        'skip': 'Transmisja została zakończona lub materiał niedostępny',
     }, {
         'url': 'tvp:22670268',
         'only_matching': True,
@@ -116,10 +118,13 @@ class TVPEmbedIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
 
-        error_massage = get_element_by_attribute('class', 'msg error', webpage)
-        if error_massage:
+        error = self._html_search_regex(
+            r'(?s)<p[^>]+\bclass=["\']notAvailable__text["\'][^>]*>(.+?)</p>',
+            webpage, 'error', default=None) or clean_html(
+            get_element_by_attribute('class', 'msg error', webpage))
+        if error:
             raise ExtractorError('%s said: %s' % (
-                self.IE_NAME, clean_html(error_massage)), expected=True)
+                self.IE_NAME, clean_html(error)), expected=True)
 
         title = self._search_regex(
             r'name\s*:\s*([\'"])Title\1\s*,\s*value\s*:\s*\1(?P<title>.+?)\1',

From d93083789bf9c318b18d52ac132e9495345b9ebc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 14:09:30 +0700
Subject: [PATCH 427/510] [tvp:series] Fix extraction

---
 youtube_dl/extractor/tvp.py | 67 ++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 536b580fc..05669a366 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,14 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     clean_html,
-    get_element_by_attribute,
+    determine_ext,
     ExtractorError,
+    get_element_by_attribute,
+    orderedSet,
 )
 
 
@@ -198,46 +200,35 @@ class TVPEmbedIE(InfoExtractor):
 
 class TVPSeriesIE(InfoExtractor):
     IE_NAME = 'tvp:series'
-    _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://vod\.tvp\.pl/website/(?P<display_id>[^,]+),(?P<id>\d+)/video'
 
     _TESTS = [{
-        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
+        'url': 'https://vod.tvp.pl/website/lzy-cennet,38678312/video',
         'info_dict': {
-            'title': 'Ogniem i mieczem',
-            'id': '4278026',
+            'id': '38678312',
         },
-        'playlist_count': 4,
-    }, {
-        'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
-        'info_dict': {
-            'title': 'Boso przez świat',
-            'id': '9329207',
-        },
-        'playlist_count': 86,
+        'playlist_count': 115,
     }]
 
+    def _entries(self, url, display_id):
+        for page_num in itertools.count(1):
+            page = self._download_webpage(
+                url, display_id, 'Downloading page %d' % page_num,
+                query={'page': page_num})
+
+            video_ids = orderedSet(re.findall(
+                r'<a[^>]+\bhref=["\']/video/%s,[^,]+,(\d+)' % display_id,
+                page))
+
+            if not video_ids:
+                break
+
+            for video_id in video_ids:
+                yield self.url_result(
+                    'tvp:%s' % video_id, ie=TVPEmbedIE.ie_key(),
+                    video_id=video_id)
+
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id, tries=5)
-
-        title = self._html_search_regex(
-            r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
-        playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
-        playlist = self._download_webpage(
-            'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
-            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5,
-            note='Downloading playlist')
-
-        videos_paths = re.findall(
-            '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
-        entries = [
-            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TVPIE.ie_key())
-            for v_path in videos_paths]
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'display_id': display_id,
-            'title': title,
-            'entries': entries,
-        }
+        mobj = re.match(self._VALID_URL, url)
+        display_id, playlist_id = mobj.group('display_id', 'id')
+        return self.playlist_result(self._entries(url, display_id), playlist_id)

From 388cfbd3d8915ebb99714ac8e7ce4151edf96d8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Feb 2019 14:27:00 +0700
Subject: [PATCH 428/510] [tvp:website] Improve support

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/tvp.py        | 26 ++++++++++++++++++++++----
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c70452dcd..923dfe7f4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1218,7 +1218,7 @@ from .tvnow import (
 from .tvp import (
     TVPEmbedIE,
     TVPIE,
-    TVPSeriesIE,
+    TVPWebsiteIE,
 )
 from .tvplay import (
     TVPlayIE,
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 05669a366..accff75b5 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -198,19 +198,36 @@ class TVPEmbedIE(InfoExtractor):
         }
 
 
-class TVPSeriesIE(InfoExtractor):
+class TVPWebsiteIE(InfoExtractor):
     IE_NAME = 'tvp:series'
-    _VALID_URL = r'https?://vod\.tvp\.pl/website/(?P<display_id>[^,]+),(?P<id>\d+)/video'
+    _VALID_URL = r'https?://vod\.tvp\.pl/website/(?P<display_id>[^,]+),(?P<id>\d+)'
 
     _TESTS = [{
+        # series
         'url': 'https://vod.tvp.pl/website/lzy-cennet,38678312/video',
         'info_dict': {
             'id': '38678312',
         },
         'playlist_count': 115,
+    }, {
+        # film
+        'url': 'https://vod.tvp.pl/website/gloria,35139666',
+        'info_dict': {
+            'id': '36637049',
+            'ext': 'mp4',
+            'title': 'Gloria, Gloria',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['TVPEmbed'],
+    }, {
+        'url': 'https://vod.tvp.pl/website/lzy-cennet,38678312',
+        'only_matching': True,
     }]
 
-    def _entries(self, url, display_id):
+    def _entries(self, display_id, playlist_id):
+        url = 'https://vod.tvp.pl/website/%s,%s/video' % (display_id, playlist_id)
         for page_num in itertools.count(1):
             page = self._download_webpage(
                 url, display_id, 'Downloading page %d' % page_num,
@@ -231,4 +248,5 @@ class TVPSeriesIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id, playlist_id = mobj.group('display_id', 'id')
-        return self.playlist_result(self._entries(url, display_id), playlist_id)
+        return self.playlist_result(
+            self._entries(display_id, playlist_id), playlist_id)

From c76fc5b22a70f9ac24fe7e34c37aa8ef82e85c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Feb 2019 02:10:06 +0700
Subject: [PATCH 429/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 398528f76..adbdf166d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Extractors
+* [tvp:website] Fix and improve extraction
++ [tvp] Detect unavailable videos
+* [tvp] Fix description extraction and make thumbnail optional
++ [linuxacademy] Add support for linuxacademy.com (#12207)
+* [bilibili] Update keys (#19233)
+* [udemy] Extend URL regular expressions (#14330, #15883)
+* [udemy] Update User-Agent and detect captcha (#14713, #15839, #18126)
+* [noovo] Fix extraction (#19230)
+* [rai] Relax URL regular expression (#19232)
++ [vshare] Pass Referer to download request (#19205, #19221)
++ [openload] Add support for oload.live (#19222)
+* [imgur] Use video id as title fallback (#18590)
++ [twitch] Add new source format detection approach (#19193)
+* [tvplayhome] Fix video id extraction (#19190)
+* [tvplayhome] Fix episode metadata extraction (#19190)
+* [rutube:embed] Fix extraction (#19163)
++ [rutube:embed] Add support private videos (#19163)
++ [soundcloud] Extract more metadata
++ [trunews] Add support for trunews.com (#19153)
++ [linkedin:learning] Extract chapter_number and chapter_id (#19162)
+
+
 version 2019.02.08
 
 Core

From 77a842c8926625fe791ed36613f183bb195394cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Feb 2019 02:11:11 +0700
Subject: [PATCH 430/510] release 2019.02.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7128d998f..ff626883d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.02.08*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.02.08**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.02.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.02.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.02.08
+[debug] youtube-dl version 2019.02.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index adbdf166d..f9dd7928f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.02.18
 
 Extractors
 * [tvp:website] Fix and improve extraction
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 32fe6b647..d8a8d7ede 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -458,6 +458,7 @@
  - **LineTV**
  - **linkedin:learning**
  - **linkedin:learning:course**
+ - **LinuxAcademy**
  - **LiTV**
  - **LiveLeak**
  - **LiveLeakEmbed**
@@ -915,6 +916,7 @@
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
+ - **TruNews**
  - **TruTV**
  - **Tube8**
  - **TubiTv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4dc5a611e..ea1d5a4a5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.02.08'
+__version__ = '2019.02.18'

From caf48f557a8f4f904c88346bcfc462069b8745bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Feb 2019 05:59:07 +0700
Subject: [PATCH 431/510] [metacafe] Fix family filter bypass (closes #19287)

---
 youtube_dl/extractor/metacafe.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 28f59f63c..9e92416d1 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
+    compat_urllib_parse,
     compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     determine_ext,
@@ -144,7 +145,7 @@ class MetacafeIE(InfoExtractor):
 
         headers = {
             # Disable family filter
-            'Cookie': 'user=%s; ' % compat_urllib_parse_urlencode({'ffilter': False})
+            'Cookie': 'user=%s; ' % compat_urllib_parse.quote(json.dumps({'ffilter': False}))
         }
 
         # AnyClip videos require the flashversion cookie so that we get the link

From 37b239b3b66ea9e2a71bae41e9da6dba8ee5554c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Feb 2019 00:43:29 +0700
Subject: [PATCH 432/510] [downloader/external] Fix infinite retries for curl
 (closes #19303)

---
 youtube_dl/downloader/external.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 958d00aac..0b88bfd94 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -121,7 +121,11 @@ class CurlFD(ExternalFD):
         cmd += self._valueless_option('--silent', 'noprogress')
         cmd += self._valueless_option('--verbose', 'verbose')
         cmd += self._option('--limit-rate', 'ratelimit')
-        cmd += self._option('--retry', 'retries')
+        retry = self._option('--retry', 'retries')
+        if len(retry) == 2:
+            if retry[1] in ('inf', 'infinite'):
+                retry[1] = '2147483647'
+            cmd += retry
         cmd += self._option('--max-filesize', 'max_filesize')
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--proxy', 'proxy')

From 8c80603f1adea843d96c0598b902106c7a3efb7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Feb 2019 00:58:56 +0700
Subject: [PATCH 433/510] [downloader/external] Add support for rate limit and
 retries for wget

---
 youtube_dl/downloader/external.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 0b88bfd94..22e6093b3 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -164,6 +164,12 @@ class WgetFD(ExternalFD):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._option('--limit-rate', 'ratelimit')
+        retry = self._option('--tries', 'retries')
+        if len(retry) == 2:
+            if retry[1] in ('inf', 'infinite'):
+                retry[1] = '0'
+            cmd += retry
         cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
         cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')

From f0228f56fb2441510aa966ba9298e388b209cde1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Feb 2019 21:01:25 +0700
Subject: [PATCH 434/510] [bbccouk] Make subtitles non fatal (#19651)

---
 youtube_dl/extractor/bbc.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index eac9a5a46..13340ec64 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1,8 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import itertools
+import re
+import xml
 
 from .common import InfoExtractor
 from ..utils import (
@@ -17,6 +18,7 @@ from ..utils import (
     parse_iso8601,
     try_get,
     unescapeHTML,
+    url_or_none,
     urlencode_postdata,
     urljoin,
 )
@@ -310,7 +312,13 @@ class BBCCoUkIE(InfoExtractor):
     def _get_subtitles(self, media, programme_id):
         subtitles = {}
         for connection in self._extract_connections(media):
-            captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
+            cc_url = url_or_none(connection.get('href'))
+            if not cc_url:
+                continue
+            captions = self._download_xml(
+                cc_url, programme_id, 'Downloading captions', fatal=False)
+            if not isinstance(captions, xml.etree.ElementTree.Element):
+                continue
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
             subtitles[lang] = [
                 {

From 55b8588f0e4dd9597b6da5c46d05b9dd1e9f5960 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Feb 2019 23:19:15 +0700
Subject: [PATCH 435/510] [servus] Fix extraction (closes #19297)

---
 youtube_dl/extractor/servus.py | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/servus.py b/youtube_dl/extractor/servus.py
index 264e1dd8b..e579d42cf 100644
--- a/youtube_dl/extractor/servus.py
+++ b/youtube_dl/extractor/servus.py
@@ -1,31 +1,44 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class ServusIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?servus\.com/(?:at|de)/p/[^/]+/(?P<id>AA-\w+|\d+-\d+)'
+    _VALID_URL = r'https?://(?:www\.)?servus\.com/(?:(?:at|de)/p/[^/]+|tv/videos)/(?P<id>[aA]{2}-\w+|\d+-\d+)'
     _TESTS = [{
         'url': 'https://www.servus.com/de/p/Die-Gr%C3%BCnen-aus-Sicht-des-Volkes/AA-1T6VBU5PW1W12/',
-        'md5': '046dee641cda1c4cabe13baef3be2c1c',
+        'md5': '3e1dd16775aa8d5cbef23628cfffc1f4',
         'info_dict': {
             'id': 'AA-1T6VBU5PW1W12',
             'ext': 'mp4',
-            'title': 'Die Grünen aus Volkssicht',
-            'description': 'md5:052b5da1cb2cd7d562ef1f19be5a5cba',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'title': 'Die Grünen aus Sicht des Volkes',
+            'description': 'md5:1247204d85783afe3682644398ff2ec4',
+            'thumbnail': r're:^https?://.*\.jpg',
         }
     }, {
         'url': 'https://www.servus.com/at/p/Wie-das-Leben-beginnt/1309984137314-381415152/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.servus.com/tv/videos/aa-1t6vbu5pw1w12/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.servus.com/tv/videos/1380889096408-1235196658/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id = self._match_id(url).upper()
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        title = self._search_regex(
+            (r'videoLabel\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<h\d+[^>]+\bclass=["\']heading--(?:one|two)["\'][^>]*>(?P<title>[^<]+)'),
+            webpage, 'title', default=None,
+            group='title') or self._og_search_title(webpage)
+        title = re.sub(r'\s*-\s*Servus TV\s*$', '', title)
         description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 

From db1c3a9d3f202cc6f3fd83a2a918869e7c0d147f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Feb 2019 03:41:15 +0700
Subject: [PATCH 436/510] [periscope] Extract width and height (closes #20015)

---
 youtube_dl/extractor/periscope.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 8afe541ec..b337a56c0 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     parse_iso8601,
     unescapeHTML,
 )
@@ -75,6 +76,14 @@ class PeriscopeIE(PeriscopeBaseIE):
             'url': broadcast[image],
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
+        width = int_or_none(broadcast.get('width'))
+        height = int_or_none(broadcast.get('height'))
+
+        def add_width_and_height(f):
+            for key, val in (('width', width), ('height', height)):
+                if not f.get(key):
+                    f[key] = val
+
         video_urls = set()
         formats = []
         for format_id in ('replay', 'rtmp', 'hls', 'https_hls', 'lhls', 'lhlsweb'):
@@ -83,16 +92,21 @@ class PeriscopeIE(PeriscopeBaseIE):
                 continue
             video_urls.add(video_url)
             if format_id != 'rtmp':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     video_url, token, 'mp4',
                     entry_protocol='m3u8_native'
                     if state in ('ended', 'timed_out') else 'm3u8',
-                    m3u8_id=format_id, fatal=False))
+                    m3u8_id=format_id, fatal=False)
+                if len(m3u8_formats) == 1:
+                    add_width_and_height(m3u8_formats[0])
+                formats.extend(m3u8_formats)
                 continue
-            formats.append({
+            rtmp_format = {
                 'url': video_url,
                 'ext': 'flv' if format_id == 'rtmp' else 'mp4',
-            })
+            }
+            add_width_and_height(rtmp_format)
+            formats.append(rtmp_format)
         self._sort_formats(formats)
 
         return {

From 9d9a8676dc02101069cf5fa9862500d39352538c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Feb 2019 23:26:52 +0700
Subject: [PATCH 437/510] [francetv:site] Extend video id regex (closes #20029,
 closes #20071)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 2ffe83a78..3c4ef08a8 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -271,7 +271,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
 
         catalogue = None
         video_id = self._search_regex(
-            r'data-main-video=(["\'])(?P<id>(?:(?!\1).)+)\1',
+            r'(?:data-main-video\s*=|videoId\s*:)\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
             webpage, 'video id', default=None, group='id')
 
         if not video_id:

From ff60ec8f029d12c119855ec82d7ce9ecda388651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Mar 2019 00:47:18 +0700
Subject: [PATCH 438/510] [npo] Fix extraction (#20084)

---
 youtube_dl/extractor/npo.py | 120 +++++++++++++++++++++++++++++++++++-
 1 file changed, 117 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 5a427c396..857845d35 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -12,11 +12,16 @@ from ..utils import (
     ExtractorError,
     fix_xml_ampersands,
     int_or_none,
+    merge_dicts,
     orderedSet,
     parse_duration,
     qualities,
+    str_or_none,
     strip_jsonp,
     unified_strdate,
+    unified_timestamp,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
@@ -176,9 +181,118 @@ class NPOIE(NPOBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self._get_info(video_id)
+        try:
+            return self._get_info(url, video_id)
+        except ExtractorError:
+            return self._get_old_info(video_id)
 
-    def _get_info(self, video_id):
+    def _get_info(self, url, video_id):
+        token = self._download_json(
+            'https://www.npostart.nl/api/token', video_id,
+            'Downloading token', headers={
+                'Referer': url,
+                'X-Requested-With': 'XMLHttpRequest',
+            })['token']
+
+        player = self._download_json(
+            'https://www.npostart.nl/player/%s' % video_id, video_id,
+            'Downloading player JSON', data=urlencode_postdata({
+                'autoplay': 0,
+                'share': 1,
+                'pageUrl': url,
+                'hasAdConsent': 0,
+                '_token': token,
+            }))
+
+        player_token = player['token']
+
+        format_urls = set()
+        formats = []
+        for profile in ('hls', 'dash-widevine', 'dash-playready', 'smooth'):
+            streams = self._download_json(
+                'https://start-player.npo.nl/video/%s/streams' % video_id,
+                video_id, 'Downloading %s profile JSON' % profile, fatal=False,
+                query={
+                    'profile': profile,
+                    'quality': 'npo',
+                    'tokenId': player_token,
+                    'streamType': 'broadcast',
+                })
+            if not streams:
+                continue
+            stream = streams.get('stream')
+            if not isinstance(stream, dict):
+                continue
+            stream_url = url_or_none(stream.get('src'))
+            if not stream_url or stream_url in format_urls:
+                continue
+            format_urls.add(stream_url)
+            if stream.get('protection') is not None:
+                continue
+            stream_type = stream.get('type')
+            stream_ext = determine_ext(stream_url)
+            if stream_type == 'application/dash+xml' or stream_ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    stream_url, video_id, mpd_id='dash', fatal=False))
+            elif stream_type == 'application/vnd.apple.mpegurl' or stream_ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url, video_id, ext='mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+            elif '.ism/Manifest' in stream_url:
+                formats.extend(self._extract_ism_formats(
+                    stream_url, video_id, ism_id='mss', fatal=False))
+            else:
+                formats.append({
+                    'url': stream_url,
+                })
+
+        self._sort_formats(formats)
+
+        info = {
+            'id': video_id,
+            'title': video_id,
+            'formats': formats,
+        }
+
+        embed_url = url_or_none(player.get('embedUrl'))
+        if embed_url:
+            webpage = self._download_webpage(
+                embed_url, video_id, 'Downloading embed page', fatal=False)
+            if webpage:
+                video = self._parse_json(
+                    self._search_regex(
+                        r'\bvideo\s*=\s*({.+?})\s*;', webpage, 'video',
+                        default='{}'), video_id)
+                if video:
+                    title = video.get('episodeTitle')
+                    subtitles = {}
+                    subtitles_list = video.get('subtitles')
+                    if isinstance(subtitles_list, list):
+                        for cc in subtitles_list:
+                            cc_url = url_or_none(cc.get('src'))
+                            if not cc_url:
+                                continue
+                            lang = str_or_none(cc.get('language')) or 'nl'
+                            subtitles.setdefault(lang, []).append({
+                                'url': cc_url,
+                            })
+                    return merge_dicts({
+                        'title': title,
+                        'description': video.get('description'),
+                        'thumbnail': url_or_none(
+                            video.get('still_image_url') or video.get('orig_image_url')),
+                        'duration': int_or_none(video.get('duration')),
+                        'timestamp': unified_timestamp(video.get('broadcastDate')),
+                        'creator': video.get('channel'),
+                        'series': video.get('title'),
+                        'episode': title,
+                        'episode_number': int_or_none(video.get('episodeNumber')),
+                        'subtitles': subtitles,
+                    }, info)
+
+        return info
+
+    def _get_old_info(self, video_id):
         metadata = self._download_json(
             'http://e.omroep.nl/metadata/%s' % video_id,
             video_id,
@@ -280,7 +394,7 @@ class NPOIE(NPOBaseIE):
             # JSON
             else:
                 video_url = stream_info.get('url')
-            if not video_url or video_url in urls:
+            if not video_url or 'vodnotavailable.' in video_url or video_url in urls:
                 continue
             urls.add(video_url)
             if determine_ext(video_url) == 'm3u8':

From 333f617b1207cb53efaa5e2f7af174cfa87deee1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Mar 2019 01:02:36 +0700
Subject: [PATCH 439/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f9dd7928f..f717f99a8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Core
++ [downloader/external] Add support for rate limit and retries for wget
+* [downloader/external] Fix infinite retries for curl (#19303)
+
+Extractors
+* [npo] Fix extraction (#20084)
+* [francetv:site] Extend video id regex (#20029, #20071)
++ [periscope] Extract width and height (#20015)
+* [servus] Fix extraction (#19297)
+* [bbccouk] Make subtitles non fatal (#19651)
+* [metacafe] Fix family filter bypass (#19287)
+
+
 version 2019.02.18
 
 Extractors

From 04c33bdfb3cd73e71bf0788f02998cab30cf1da2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Mar 2019 01:03:51 +0700
Subject: [PATCH 440/510] release 2019.03.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ff626883d..71a500f04 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.02.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.02.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.03.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.02.18
+[debug] youtube-dl version 2019.03.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index f717f99a8..018a30641 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.03.01
 
 Core
 + [downloader/external] Add support for rate limit and retries for wget
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ea1d5a4a5..42ba37f15 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.02.18'
+__version__ = '2019.03.01'

From 06242d44fe261999e2424d9ecb00f20ff30ccb9b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 1 Mar 2019 08:14:34 +0100
Subject: [PATCH 441/510] [vimeo] add support for Vimeo Pro portfolio protected
 videos(closes #20070)

---
 youtube_dl/extractor/vimeo.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6215b3258..6f32ea6f1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -502,7 +502,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         orig_url = url
-        if mobj.group('pro') or mobj.group('player'):
+        if mobj.group('pro'):
+            # some videos require portfolio_id to be present in player url
+            # https://github.com/rg3/youtube-dl/issues/20070
+            url = self._extract_url(url, self._download_webpage(url, video_id))
+        elif mobj.group('player'):
             url = 'https://player.vimeo.com/video/' + video_id
         elif any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
             url = 'https://vimeo.com/' + video_id

From c5b02efe20cff1612104fd731c7f02cbbce4f5f3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 1 Mar 2019 15:08:11 +0100
Subject: [PATCH 442/510] [sixplay] handle videos with empty assets(closes
 #20016)

---
 youtube_dl/extractor/sixplay.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index 0c4f865ef..35bc9fa50 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -61,7 +61,8 @@ class SixPlayIE(InfoExtractor):
         quality_key = qualities(['lq', 'sd', 'hq', 'hd'])
         formats = []
         subtitles = {}
-        for asset in clip_data['assets']:
+        assets = clip_data.get('assets') or []
+        for asset in assets:
             asset_url = asset.get('full_physical_path')
             protocol = asset.get('protocol')
             if not asset_url or protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264' or asset_url in urls:

From 398e1e21d6cbf6eb1e8e7e84de4fad30b7d59613 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 1 Mar 2019 15:34:05 +0100
Subject: [PATCH 443/510] [espn] extend _VALID_URL regex(closes #20013)

---
 youtube_dl/extractor/espn.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 127c69b2e..8cc9bd165 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -29,7 +29,8 @@ class ESPNIE(OnceIE):
                                     (?:
                                         .*?\?.*?\bid=|
                                         /_/id/
-                                    )
+                                    )|
+                                    [^/]+/video/
                                 )
                             )|
                             (?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
@@ -94,6 +95,9 @@ class ESPNIE(OnceIE):
     }, {
         'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
         'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/espnw/video/26066627/arkansas-gibson-completes-hr-cycle-four-innings',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From dca0e0040ae97b2fc0cd54d5e819a5a278937350 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Mar 2019 08:01:42 +0100
Subject: [PATCH 444/510] Revert "use older login method(closes #11572)"

This reverts commit cc6a960e134614f8af2a42dcd8bf146d63638a3c.
---
 youtube_dl/extractor/crunchyroll.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 5e2cbe41d..ce2e2d3ba 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -56,17 +56,6 @@ class CrunchyrollBaseIE(InfoExtractor):
         if username is None:
             return
 
-        self._download_webpage(
-            'https://www.crunchyroll.com/?a=formhandler',
-            None, 'Logging in', 'Wrong login info',
-            data=urlencode_postdata({
-                'formname': 'RpcApiUser_Login',
-                'next_url': 'https://www.crunchyroll.com/acct/membership',
-                'name': username,
-                'password': password,
-            }))
-
-        '''
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
@@ -110,7 +99,6 @@ class CrunchyrollBaseIE(InfoExtractor):
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
         raise ExtractorError('Unable to log in')
-        '''
 
     def _real_initialize(self):
         self._login()

From a8f83f0c56e81b871a46c18fa9ebc6643370fa48 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Mar 2019 08:25:47 +0100
Subject: [PATCH 445/510] [crunchyroll] fix is_logged check

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index ce2e2d3ba..fd1e7afad 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -60,7 +60,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             self._LOGIN_URL, None, 'Downloading login page')
 
         def is_logged(webpage):
-            return '<title>Redirecting' in webpage
+            return 'href="/logout"' in webpage
 
         # Already logged in
         if is_logged(login_page):

From 7465e0aee2301c3e86fe38d6e0ef5ad01c16ec79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Mar 2019 06:25:45 +0700
Subject: [PATCH 446/510] [spankbang] Fix extraction (closes #20023)

---
 youtube_dl/extractor/spankbang.py | 45 +++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index fbe6ef31a..f11d728ca 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -9,6 +9,8 @@ from ..utils import (
     parse_duration,
     parse_resolution,
     str_to_int,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
@@ -64,16 +66,49 @@ class SpankBangIE(InfoExtractor):
                 'Video %s is not available' % video_id, expected=True)
 
         formats = []
-        for mobj in re.finditer(
-                r'stream_url_(?P<id>[^\s=]+)\s*=\s*(["\'])(?P<url>(?:(?!\2).)+)\2',
-                webpage):
-            format_id, format_url = mobj.group('id', 'url')
+
+        def extract_format(format_id, format_url):
+            f_url = url_or_none(format_url)
+            if not f_url:
+                return
             f = parse_resolution(format_id)
             f.update({
-                'url': format_url,
+                'url': f_url,
                 'format_id': format_id,
             })
             formats.append(f)
+
+        STREAM_URL_PREFIX = 'stream_url_'
+
+        for mobj in re.finditer(
+                r'%s(?P<id>[^\s=]+)\s*=\s*(["\'])(?P<url>(?:(?!\2).)+)\2'
+                % STREAM_URL_PREFIX, webpage):
+            extract_format(mobj.group('id', 'url'))
+
+        if not formats:
+            stream_key = self._search_regex(
+                r'data-streamkey\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
+                webpage, 'stream key', group='value')
+
+            sb_csrf_session = self._get_cookies(
+                'https://spankbang.com')['sb_csrf_session'].value
+
+            stream = self._download_json(
+                'https://spankbang.com/api/videos/stream', video_id,
+                'Downloading stream JSON', data=urlencode_postdata({
+                    'id': stream_key,
+                    'data': 0,
+                    'sb_csrf_session': sb_csrf_session,
+                }), headers={
+                    'Referer': url,
+                    'X-CSRFToken': sb_csrf_session,
+                })
+
+            for format_id, format_url in stream.items():
+                if format_id.startswith(STREAM_URL_PREFIX):
+                    extract_format(
+                        format_id[len(STREAM_URL_PREFIX):], format_url)
+
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From 7aeb788e564d397face83b580362189753edd9dd Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@me.com>
Date: Sun, 3 Mar 2019 02:16:48 +0100
Subject: [PATCH 447/510] [travis] Remove sudo: false

Travis now recommends removing `sudo: false` from configuration: https://blog.travis-ci.com/2018-11-19-required-linux-infrastructure-migration.
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 79287ccf6..82e81d078 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,6 @@ python:
   - "3.6"
   - "pypy"
   - "pypy3"
-sudo: false
 env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download

From 8ae113ca9df0abd790e3391cd529bac42fce304f Mon Sep 17 00:00:00 2001
From: dimqua <dimqua@users.noreply.github.com>
Date: Sun, 3 Mar 2019 04:19:36 +0300
Subject: [PATCH 448/510] [youtube] Add more invidious instances

See [Invidious-Instances](https://github.com/omarroth/invidious/wiki/Invidious-Instances) for the reference.
---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c8bf98b58..457e2acea 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -352,6 +352,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             (?:www\.)?yourepeat\.com/|
                             tube\.majestyc\.net/|
                             (?:www\.)?invidio\.us/|
+                            (?:www\.)?invidious\.snopyta\.org/|
+                            (?:www\.)?invidious\.kabi\.tk/|
+                            (?:www\.)?vid\.wxzm\.sx/|
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:

From 0a5baf9c210df9f492ae48dd8fdae90561c971bd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Mar 2019 06:18:15 +0100
Subject: [PATCH 449/510] [libsyn] improve extraction(closes #20229)

---
 youtube_dl/extractor/libsyn.py | 64 +++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index f7311f483..2cf444258 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
+    get_element_by_class,
     parse_duration,
+    strip_or_none,
     unified_strdate,
 )
 
@@ -21,7 +23,9 @@ class LibsynIE(InfoExtractor):
             'id': '6385796',
             'ext': 'mp3',
             'title': "Champion Minded - Developing a Growth Mindset",
-            'description': 'In this episode, Allistair talks about the importance of developing a growth mindset, not only in sports, but in life too.',
+            # description fetched using another request:
+            # http://html5-player.libsyn.com/embed/getitemdetails?item_id=6385796
+            # 'description': 'In this episode, Allistair talks about the importance of developing a growth mindset, not only in sports, but in life too.',
             'upload_date': '20180320',
             'thumbnail': 're:^https?://.*',
         },
@@ -38,22 +42,36 @@ class LibsynIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-        url = m.group('mainurl')
+        url, video_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, video_id)
 
-        podcast_title = self._search_regex(
-            r'<h3>([^<]+)</h3>', webpage, 'podcast title', default=None)
-        if podcast_title:
-            podcast_title = podcast_title.strip()
-        episode_title = self._search_regex(
-            r'(?:<div class="episode-title">|<h4>)([^<]+)</', webpage, 'episode title')
-        if episode_title:
-            episode_title = episode_title.strip()
+        data = self._parse_json(self._search_regex(
+            r'var\s+playlistItem\s*=\s*({.+?});',
+            webpage, 'JSON data block'), video_id)
+
+        episode_title = data.get('item_title') or get_element_by_class('episode-title', webpage)
+        if not episode_title:
+            self._search_regex(
+                [r'data-title="([^"]+)"', r'<title>(.+?)</title>'],
+                webpage, 'episode title')
+        episode_title = episode_title.strip()
+
+        podcast_title = strip_or_none(clean_html(self._search_regex(
+            r'<h3>([^<]+)</h3>', webpage, 'podcast title',
+            default=None) or get_element_by_class('podcast-title', webpage)))
 
         title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title
 
+        formats = []
+        for k, format_id in (('media_url_libsyn', 'libsyn'), ('media_url', 'main'), ('download_link', 'download')):
+            f_url = data.get(k)
+            if not f_url:
+                continue
+            formats.append({
+                'url': f_url,
+                'format_id': format_id,
+            })
+
         description = self._html_search_regex(
             r'<p\s+id="info_text_body">(.+?)</p>', webpage,
             'description', default=None)
@@ -61,27 +79,15 @@ class LibsynIE(InfoExtractor):
             # Strip non-breaking and normal spaces
             description = description.replace('\u00A0', ' ').strip()
         release_date = unified_strdate(self._search_regex(
-            r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))
-
-        data_json = self._search_regex(r'var\s+playlistItem\s*=\s*(\{.*?\});\n', webpage, 'JSON data block')
-        data = json.loads(data_json)
-
-        formats = [{
-            'url': data['media_url'],
-            'format_id': 'main',
-        }, {
-            'url': data['media_url_libsyn'],
-            'format_id': 'libsyn',
-        }]
-        thumbnail = data.get('thumbnail_url')
-        duration = parse_duration(data.get('duration'))
+            r'<div class="release_date">Released: ([^<]+)<',
+            webpage, 'release date', default=None) or data.get('release_date'))
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnail': data.get('thumbnail_url'),
             'upload_date': release_date,
-            'duration': duration,
+            'duration': parse_duration(data.get('duration')),
             'formats': formats,
         }

From e7e62441cdde6dca6211c073be73677f195a0dff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 3 Mar 2019 13:23:59 +0100
Subject: [PATCH 450/510] [utils] strip #HttpOnly_ prefix from cookies files
 (#20219)

---
 test/test_YoutubeDLCookieJar.py            | 10 ++++++++++
 test/testdata/cookies/httponly_cookies.txt |  6 ++++++
 youtube_dl/utils.py                        | 18 +++++++++++++++++-
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 test/testdata/cookies/httponly_cookies.txt

diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py
index 6a8243590..f959798de 100644
--- a/test/test_YoutubeDLCookieJar.py
+++ b/test/test_YoutubeDLCookieJar.py
@@ -29,6 +29,16 @@ class TestYoutubeDLCookieJar(unittest.TestCase):
             tf.close()
             os.remove(tf.name)
 
+    def test_strip_httponly_prefix(self):
+        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
+        cookiejar.load(ignore_discard=True, ignore_expires=True)
+
+        def assert_cookie_has_value(key):
+            self.assertEqual(cookiejar._cookies['www.foobar.foobar']['/'][key].value, key + '_VALUE')
+
+        assert_cookie_has_value('HTTPONLY_COOKIE')
+        assert_cookie_has_value('JS_ACCESSIBLE_COOKIE')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/testdata/cookies/httponly_cookies.txt b/test/testdata/cookies/httponly_cookies.txt
new file mode 100644
index 000000000..c46541d6b
--- /dev/null
+++ b/test/testdata/cookies/httponly_cookies.txt
@@ -0,0 +1,6 @@
+# Netscape HTTP Cookie File
+# http://curl.haxx.se/rfc/cookie_spec.html
+# This is a generated file!  Do not edit.
+
+#HttpOnly_www.foobar.foobar	FALSE	/	TRUE	2147483647	HTTPONLY_COOKIE	HTTPONLY_COOKIE_VALUE
+www.foobar.foobar	FALSE	/	TRUE	2147483647	JS_ACCESSIBLE_COOKIE	JS_ACCESSIBLE_COOKIE_VALUE
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f5a0bb4b0..a71eda85d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1141,6 +1141,8 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
 
 
 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
+    _HTTPONLY_PREFIX = '#HttpOnly_'
+
     def save(self, filename=None, ignore_discard=False, ignore_expires=False):
         # Store session cookies with `expires` set to 0 instead of an empty
         # string
@@ -1150,7 +1152,21 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
         compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
 
     def load(self, filename=None, ignore_discard=False, ignore_expires=False):
-        compat_cookiejar.MozillaCookieJar.load(self, filename, ignore_discard, ignore_expires)
+        """Load cookies from a file."""
+        if filename is None:
+            if self.filename is not None:
+                filename = self.filename
+            else:
+                raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
+
+        cf = io.StringIO()
+        with open(filename) as f:
+            for line in f:
+                if line.startswith(self._HTTPONLY_PREFIX):
+                    line = line[len(self._HTTPONLY_PREFIX):]
+                cf.write(compat_str(line))
+        cf.seek(0)
+        self._really_load(cf, filename, ignore_discard, ignore_expires)
         # Session cookies are denoted by either `expires` field set to
         # an empty string or 0. MozillaCookieJar only recognizes the former
         # (see [1]). So we need force the latter to be recognized as session

From 39c780fdec2c62135f37e3565efedf7dcad605ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Mar 2019 00:37:39 +0700
Subject: [PATCH 451/510] [extractor/common] Return MPD manifest as format's
 url meta field (#20242) For symmetry with other segmented media

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c3b0586a0..1fa8048b8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2120,7 +2120,7 @@ class InfoExtractor(object):
                         bandwidth = int_or_none(representation_attrib.get('bandwidth'))
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
-                            'url': base_url,
+                            'url': mpd_url,
                             'manifest_url': mpd_url,
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),

From c790e93ab5db5f318fb094b8a45f9160cdf4bd9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Mar 2019 00:39:15 +0700
Subject: [PATCH 452/510] [extractor/common] Clarify url and manifest_url meta
 fields

---
 youtube_dl/extractor/common.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1fa8048b8..641e50f3c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -102,10 +102,20 @@ class InfoExtractor(object):
                     from worst to best quality.
 
                     Potential fields:
-                    * url        Mandatory. The URL of the video file
+                    * url        The mandatory URL representing the media:
+                                   for plain file media - HTTP URL of this file,
+                                   for RTMP - RTMP URL,
+                                   for HLS - URL of the M3U8 media playlist,
+                                   for HDS - URL of the F4M manifest,
+                                   for DASH - URL of the MPD manifest,
+                                   for MSS - URL of the ISM manifest.
                     * manifest_url
                                  The URL of the manifest file in case of
-                                 fragmented media (DASH, hls, hds)
+                                 fragmented media:
+                                   for HLS - URL of the M3U8 master playlist,
+                                   for HDS - URL of the F4M manifest,
+                                   for DASH - URL of the MPD manifest,
+                                   for MSS - URL of the ISM manifest.
                     * ext        Will be calculated from URL if missing
                     * format     A human-readable description of the format
                                  ("mp4 container with h264/opus").

From 5dcd630dca9b75ec2ca920ae7799252e0e0bb599 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Mar 2019 22:26:32 +0100
Subject: [PATCH 453/510] [paramountnetwork] fix mgid extraction(closes #20241)

---
 youtube_dl/extractor/spike.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 6090e0066..21b93a5b3 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -46,8 +46,12 @@ class ParamountNetworkIE(MTVServicesInfoExtractor):
     _GEO_COUNTRIES = ['US']
 
     def _extract_mgid(self, webpage):
-        cs = self._parse_json(self._search_regex(
+        root_data = self._parse_json(self._search_regex(
             r'window\.__DATA__\s*=\s*({.+})',
-            webpage, 'data'), None)['children']
-        c = next(c for c in cs if c.get('type') == 'VideoPlayer')
+            webpage, 'data'), None)
+
+        def find_sub_data(data, data_type):
+            return next(c for c in data['children'] if c.get('type') == data_type)
+
+        c = find_sub_data(find_sub_data(root_data, 'MainContainer'), 'VideoPlayer')
         return c['props']['media']['video']['config']['uri']

From d9eb580a796ef6c9a248fdd8896ccf85349c35eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Mar 2019 23:45:40 +0700
Subject: [PATCH 454/510] [extractor/common] Do not fail on invalid data while
 parsing F4M manifest in non fatal mode

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 641e50f3c..55ce1a888 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -13,6 +13,7 @@ import socket
 import sys
 import time
 import math
+import xml
 
 from ..compat import (
     compat_cookiejar,
@@ -1464,6 +1465,9 @@ class InfoExtractor(object):
     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
                            fatal=True, m3u8_id=None):
+        if not isinstance(manifest, xml.etree.ElementTree.Element) and not fatal:
+            return []
+
         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
         if akamai_pv is not None and ';' in akamai_pv.text:

From c17eb5b4b06cfa2c8bffb378b0a5c84d4c5a6834 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Mar 2019 23:54:25 +0700
Subject: [PATCH 455/510] [rai] Improve extraction (closes #20253)

---
 youtube_dl/extractor/rai.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 149153b8f..207a6c247 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -74,11 +74,11 @@ class RaiBaseIE(InfoExtractor):
             if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
                 continue
 
-            if ext == 'm3u8':
+            if ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
                 formats.extend(self._extract_m3u8_formats(
                     media_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id='hls', fatal=False))
-            elif ext == 'f4m':
+            elif ext == 'f4m' or platform == 'flash':
                 manifest_url = update_url_query(
                     media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
                     {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})

From bb6f112d9d57d7c6260de132cad604c1c05bc5a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Mar 2019 23:57:39 +0700
Subject: [PATCH 456/510] [npo] Improve ISM extraction

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 857845d35..ad62f8ec6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -238,7 +238,7 @@ class NPOIE(NPOBaseIE):
                 formats.extend(self._extract_m3u8_formats(
                     stream_url, video_id, ext='mp4',
                     entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
-            elif '.ism/Manifest' in stream_url:
+            elif re.search(r'\.isml?/Manifest', stream_url):
                 formats.extend(self._extract_ism_formats(
                     stream_url, video_id, ism_id='mss', fatal=False))
             else:

From e5ada4f3ad771d4cf3f533efb2597a3f1618ce75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Mar 2019 00:33:08 +0700
Subject: [PATCH 457/510] [extractor/common] Fallback url to base URL for DASH
 formats

---
 youtube_dl/extractor/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 55ce1a888..a17f7cbc4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,7 +108,10 @@ class InfoExtractor(object):
                                    for RTMP - RTMP URL,
                                    for HLS - URL of the M3U8 media playlist,
                                    for HDS - URL of the F4M manifest,
-                                   for DASH - URL of the MPD manifest,
+                                   for DASH - URL of the MPD manifest or
+                                              base URL representing the media
+                                              if MPD manifest is parsed from
+                                              a string,
                                    for MSS - URL of the ISM manifest.
                     * manifest_url
                                  The URL of the manifest file in case of
@@ -2134,7 +2137,8 @@ class InfoExtractor(object):
                         bandwidth = int_or_none(representation_attrib.get('bandwidth'))
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
-                            'url': mpd_url,
+                            # NB: mpd_url may be empty when MPD manifest is parsed from a string
+                            'url': mpd_url or base_url,
                             'manifest_url': mpd_url,
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),

From 399f76870d7dc72631e7da1f54a46ed8a039c838 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Mar 2019 01:18:52 +0700
Subject: [PATCH 458/510] [compat] Introduce compat_etree_Element

---
 test/test_compat.py  |  7 +++++++
 youtube_dl/compat.py | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 51fe6aa0b..4822260ac 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -13,6 +13,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from youtube_dl.compat import (
     compat_getenv,
     compat_setenv,
+    compat_etree_Element,
     compat_etree_fromstring,
     compat_expanduser,
     compat_shlex_split,
@@ -90,6 +91,12 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag'])
         self.assertEqual(compat_shlex_split('-val 中文'), ['-val', '中文'])
 
+    def test_compat_etree_Element(self):
+        try:
+            compat_etree_Element.text
+        except AttributeError:
+            self.fail('compat_etree_Element is not a type')
+
     def test_compat_etree_fromstring(self):
         xml = '''
             <root foo="bar" spam="中文">
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 7b770340f..b2fe62f12 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2508,6 +2508,15 @@ class _TreeBuilder(etree.TreeBuilder):
         pass
 
 
+try:
+    # xml.etree.ElementTree.Element is a method in Python <=2.6 and
+    # the following will crash with:
+    #  TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types
+    isinstance(None, xml.etree.ElementTree.Element)
+    from xml.etree.ElementTree import Element as compat_etree_Element
+except TypeError:  # Python <=2.6
+    from xml.etree.ElementTree import _ElementInterface as compat_etree_Element
+
 if sys.version_info[0] >= 3:
     def compat_etree_fromstring(text):
         return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
@@ -2969,6 +2978,7 @@ __all__ = [
     'compat_cookiejar',
     'compat_cookies',
     'compat_ctypes_WINFUNCTYPE',
+    'compat_etree_Element',
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',

From ee0ba927aac067dec533a618540e43ed3deebaba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Mar 2019 01:21:57 +0700
Subject: [PATCH 459/510] Use compat_etree_Element

---
 youtube_dl/extractor/bbc.py         |  4 ++--
 youtube_dl/extractor/common.py      |  8 ++++----
 youtube_dl/extractor/crunchyroll.py | 10 +++++-----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 13340ec64..d479d2577 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 import itertools
 import re
-import xml
 
 from .common import InfoExtractor
 from ..utils import (
@@ -23,6 +22,7 @@ from ..utils import (
     urljoin,
 )
 from ..compat import (
+    compat_etree_Element,
     compat_HTTPError,
     compat_urlparse,
 )
@@ -317,7 +317,7 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             captions = self._download_xml(
                 cc_url, programme_id, 'Downloading captions', fatal=False)
-            if not isinstance(captions, xml.etree.ElementTree.Element):
+            if not isinstance(captions, compat_etree_Element):
                 continue
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
             subtitles[lang] = [
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a17f7cbc4..4839edbf7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -13,11 +13,11 @@ import socket
 import sys
 import time
 import math
-import xml
 
 from ..compat import (
     compat_cookiejar,
     compat_cookies,
+    compat_etree_Element,
     compat_etree_fromstring,
     compat_getpass,
     compat_integer_types,
@@ -802,7 +802,7 @@ class InfoExtractor(object):
             fatal=True, encoding=None, data=None, headers={}, query={},
             expected_status=None):
         """
-        Return a tuple (xml as an xml.etree.ElementTree.Element, URL handle).
+        Return a tuple (xml as an compat_etree_Element, URL handle).
 
         See _download_webpage docstring for arguments specification.
         """
@@ -823,7 +823,7 @@ class InfoExtractor(object):
             transform_source=None, fatal=True, encoding=None,
             data=None, headers={}, query={}, expected_status=None):
         """
-        Return the xml as an xml.etree.ElementTree.Element.
+        Return the xml as an compat_etree_Element.
 
         See _download_webpage docstring for arguments specification.
         """
@@ -1468,7 +1468,7 @@ class InfoExtractor(object):
     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
                            fatal=True, m3u8_id=None):
-        if not isinstance(manifest, xml.etree.ElementTree.Element) and not fatal:
+        if not isinstance(manifest, compat_etree_Element) and not fatal:
             return []
 
         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index fd1e7afad..5948154f8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 import re
 import json
-import xml.etree.ElementTree as etree
 import zlib
 
 from hashlib import sha1
@@ -12,6 +11,7 @@ from .common import InfoExtractor
 from .vrv import VRVIE
 from ..compat import (
     compat_b64decode,
+    compat_etree_Element,
     compat_etree_fromstring,
     compat_urllib_parse_urlencode,
     compat_urllib_request,
@@ -390,7 +390,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'Downloading subtitles for ' + sub_name, data={
                     'subtitle_script_id': sub_id,
                 })
-            if not isinstance(sub_doc, etree.Element):
+            if not isinstance(sub_doc, compat_etree_Element):
                 continue
             sid = sub_doc.get('id')
             iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
@@ -507,7 +507,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                         'video_quality': stream_quality,
                         'current_page': url,
                     })
-                if isinstance(streamdata, etree.Element):
+                if isinstance(streamdata, compat_etree_Element):
                     stream_info = streamdata.find('./{default}preload/stream_info')
                     if stream_info is not None:
                         stream_infos.append(stream_info)
@@ -518,7 +518,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                         'video_format': stream_format,
                         'video_encode_quality': stream_quality,
                     })
-                if isinstance(stream_info, etree.Element):
+                if isinstance(stream_info, compat_etree_Element):
                     stream_infos.append(stream_info)
                 for stream_info in stream_infos:
                     video_encode_id = xpath_text(stream_info, './video_encode_id')
@@ -593,7 +593,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         season = episode = episode_number = duration = thumbnail = None
 
-        if isinstance(metadata, etree.Element):
+        if isinstance(metadata, compat_etree_Element):
             season = xpath_text(metadata, 'series_title')
             episode = xpath_text(metadata, 'episode_title')
             episode_number = int_or_none(xpath_text(metadata, 'episode_number'))

From a551768acfd177e425f518c43a2992a50a2ff69f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Mar 2019 01:27:22 +0700
Subject: [PATCH 460/510] [facebook] Improve uploader extraction (closes
 #20250)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 74954049d..789dd79d5 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -424,7 +424,7 @@ class FacebookIE(InfoExtractor):
         uploader = clean_html(get_element_by_id(
             'fbPhotoPageAuthorName', webpage)) or self._search_regex(
             r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader',
-            fatal=False) or self._og_search_title(webpage, fatal=False)
+            default=None) or self._og_search_title(webpage, fatal=False)
         timestamp = int_or_none(self._search_regex(
             r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
             'timestamp', default=None))

From 97157c692c94e3853a6ad1b8a220f064815b6957 Mon Sep 17 00:00:00 2001
From: yonaikerlol <yonaikerlol@protonmail.com>
Date: Tue, 5 Mar 2019 14:34:34 -0400
Subject: [PATCH 461/510] [openload] Add support for oload.space

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index c1dcbb7eb..bae7c7ee7 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -249,7 +249,7 @@ class OpenloadIE(InfoExtractor):
                             (?:www\.)?
                             (?:
                                 openload\.(?:co|io|link|pw)|
-                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live)
+                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)
                             )
                         )/
                         (?:f|embed)/
@@ -349,6 +349,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.live/f/-Z58UZ-GR4M',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.space/f/IY4eZSst3u8/',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From d347b52b63282b3276815fd03fc63a1bc8b82cf5 Mon Sep 17 00:00:00 2001
From: 0x9fff00 <0x9fff00+git@protonmail.ch>
Date: Tue, 5 Mar 2019 20:11:32 +0100
Subject: [PATCH 462/510] [urplay] Extract timestamp (#20235)

---
 youtube_dl/extractor/urplay.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/urplay.py b/youtube_dl/extractor/urplay.py
index 8e6fd4731..6030b7cb5 100644
--- a/youtube_dl/extractor/urplay.py
+++ b/youtube_dl/extractor/urplay.py
@@ -2,18 +2,31 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import unified_timestamp
 
 
 class URPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://urplay.se/program/190031-tripp-trapp-trad-sovkudde',
-        'md5': 'ad5f0de86f16ca4c8062cd103959a9eb',
+        'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
+        'md5': 'ff5b0c89928f8083c74bbd5099c9292d',
+        'info_dict': {
+            'id': '203704',
+            'ext': 'mp4',
+            'title': 'UR Samtiden - Livet, universum och rymdens märkliga musik : Om vetenskap, kritiskt tänkande och motstånd',
+            'description': 'md5:5344508a52aa78c1ced6c1b8b9e44e9a',
+            'timestamp': 1513512768,
+            'upload_date': '20171217',
+        },
+    }, {
+        'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
         'info_dict': {
             'id': '190031',
             'ext': 'mp4',
             'title': 'Tripp, Trapp, Träd : Sovkudde',
             'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
+            'timestamp': 1440093600,
+            'upload_date': '20150820',
         },
     }, {
         'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
@@ -51,6 +64,7 @@ class URPlayIE(InfoExtractor):
             'title': urplayer_data['title'],
             'description': self._og_search_description(webpage),
             'thumbnail': urplayer_data.get('image'),
+            'timestamp': unified_timestamp(self._html_search_meta(('uploadDate', 'schema:uploadDate'), webpage, 'timestamp')),
             'series': urplayer_data.get('series_title'),
             'subtitles': subtitles,
             'formats': formats,

From fca9baf0da9720bac25d160924204395930191fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Mar 2019 02:45:33 +0700
Subject: [PATCH 463/510] [test] Fix test_compat_etree_Element

---
 test/test_compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 4822260ac..86ff389fd 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -93,7 +93,7 @@ class TestCompat(unittest.TestCase):
 
     def test_compat_etree_Element(self):
         try:
-            compat_etree_Element.text
+            compat_etree_Element.items
         except AttributeError:
             self.fail('compat_etree_Element is not a type')
 

From 829685b88a0c7610a874b980bc25b308c4f34590 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Mar 2019 09:20:27 +0100
Subject: [PATCH 464/510] [toutv] fix authentication(closes #20261)

---
 youtube_dl/extractor/toutv.py | 53 +++++++++--------------------------
 1 file changed, 14 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index f1ab91cf2..124ca064c 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -1,14 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
+import json
 
 from .radiocanada import RadioCanadaIE
 from ..utils import (
-    extract_attributes,
     int_or_none,
     merge_dicts,
-    urlencode_postdata,
 )
 
 
@@ -38,47 +36,24 @@ class TouTvIE(RadioCanadaIE):
         'url': 'https://ici.tou.tv/l-age-adulte/S01C501',
         'only_matching': True,
     }]
+    _CLIENT_KEY = '4dd36440-09d5-4468-8923-b6d91174ad36'
 
     def _real_initialize(self):
         email, password = self._get_login_info()
         if email is None:
             return
-        login_webpage = self._download_webpage(
-            'https://services.radio-canada.ca/auth/oauth/v2/authorize',
-            None, 'Downloading login page', query={
-                'client_id': '4dd36440-09d5-4468-8923-b6d91174ad36',
-                'redirect_uri': 'https://ici.tou.tv/logincallback',
-                'response_type': 'token',
-                'scope': 'id.write media-validation.read',
-                'state': '/',
-            })
-
-        def extract_form_url_and_data(wp, default_form_url, form_spec_re=''):
-            form, form_elem = re.search(
-                r'(?s)((<form[^>]+?%s[^>]*?>).+?</form>)' % form_spec_re, wp).groups()
-            form_data = self._hidden_inputs(form)
-            form_url = extract_attributes(form_elem).get('action') or default_form_url
-            return form_url, form_data
-
-        post_url, form_data = extract_form_url_and_data(
-            login_webpage,
-            'https://services.radio-canada.ca/auth/oauth/v2/authorize/login',
-            r'(?:id|name)="Form-login"')
-        form_data.update({
-            'login-email': email,
-            'login-password': password,
-        })
-        consent_webpage = self._download_webpage(
-            post_url, None, 'Logging in', data=urlencode_postdata(form_data))
-        post_url, form_data = extract_form_url_and_data(
-            consent_webpage,
-            'https://services.radio-canada.ca/auth/oauth/v2/authorize/consent')
-        _, urlh = self._download_webpage_handle(
-            post_url, None, 'Following Redirection',
-            data=urlencode_postdata(form_data))
-        self._access_token = self._search_regex(
-            r'access_token=([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
-            urlh.geturl(), 'access token')
+        self._access_token = self._download_json(
+            'https://services.radio-canada.ca/toutv/profiling/accounts/login',
+            None, 'Logging in', data=json.dumps({
+                'ClientId': self._CLIENT_KEY,
+                'ClientSecret': '34026772-244b-49b6-8b06-317b30ac9a20',
+                'Email': email,
+                'Password': password,
+                'Scope': 'id.write media-validation.read',
+            }).encode(), headers={
+                'Authorization': 'client-key ' + self._CLIENT_KEY,
+                'Content-Type': 'application/json;charset=utf-8',
+            })['access_token']
         self._claims = self._call_api('validation/v2/getClaims')['claims']
 
     def _real_extract(self, url):

From 7b6e76087080eac54e14cdead4e3bc0225c654b5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Mar 2019 09:28:14 +0100
Subject: [PATCH 465/510] [toutv] detect invalid login error

---
 youtube_dl/extractor/toutv.py | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 124ca064c..25e1fd46d 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -4,7 +4,9 @@ from __future__ import unicode_literals
 import json
 
 from .radiocanada import RadioCanadaIE
+from ..compat import compat_HTTPError
 from ..utils import (
+    ExtractorError,
     int_or_none,
     merge_dicts,
 )
@@ -42,18 +44,24 @@ class TouTvIE(RadioCanadaIE):
         email, password = self._get_login_info()
         if email is None:
             return
-        self._access_token = self._download_json(
-            'https://services.radio-canada.ca/toutv/profiling/accounts/login',
-            None, 'Logging in', data=json.dumps({
-                'ClientId': self._CLIENT_KEY,
-                'ClientSecret': '34026772-244b-49b6-8b06-317b30ac9a20',
-                'Email': email,
-                'Password': password,
-                'Scope': 'id.write media-validation.read',
-            }).encode(), headers={
-                'Authorization': 'client-key ' + self._CLIENT_KEY,
-                'Content-Type': 'application/json;charset=utf-8',
-            })['access_token']
+        try:
+            self._access_token = self._download_json(
+                'https://services.radio-canada.ca/toutv/profiling/accounts/login',
+                None, 'Logging in', data=json.dumps({
+                    'ClientId': self._CLIENT_KEY,
+                    'ClientSecret': '34026772-244b-49b6-8b06-317b30ac9a20',
+                    'Email': email,
+                    'Password': password,
+                    'Scope': 'id.write media-validation.read',
+                }).encode(), headers={
+                    'Authorization': 'client-key ' + self._CLIENT_KEY,
+                    'Content-Type': 'application/json;charset=utf-8',
+                })['access_token']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                error = self._parse_json(e.cause.read().decode(), None)['Message']
+                raise ExtractorError(error, expected=True)
+            raise
         self._claims = self._call_api('validation/v2/getClaims')['claims']
 
     def _real_extract(self, url):

From 9d74ea6d36696396392974e94a40dce1e5a881a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Mar 2019 23:26:59 +0700
Subject: [PATCH 466/510] [francetv:site] Relax video id regex and update test
 (closes #20268)

---
 youtube_dl/extractor/francetv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 3c4ef08a8..6101fb6bd 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -215,7 +215,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
     _TESTS = [{
         'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
         'info_dict': {
-            'id': '162311093',
+            'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1',
             'ext': 'mp4',
             'title': '13h15, le dimanche... - Les mystères de Jésus',
             'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42',
@@ -271,7 +271,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
 
         catalogue = None
         video_id = self._search_regex(
-            r'(?:data-main-video\s*=|videoId\s*:)\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
+            r'(?:data-main-video\s*=|videoId["\']?\s*[:=])\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
             webpage, 'video id', default=None, group='id')
 
         if not video_id:

From bba35695eb4ab9cc70624583375ba3d15b4e6cc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Mar 2019 02:52:08 +0700
Subject: [PATCH 467/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 018a30641..272191a01 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,34 @@
+version <unreleased>
+
+Core
+* [extractor/common] Use compat_etree_Element
++ [compat] Introduce compat_etree_Element
+* [extractor/common] Fallback url to base URL for DASH formats
+* [extractor/common] Do not fail on invalid data while parsing F4M manifest
+  in non fatal mode
+* [extractor/common] Return MPD manifest as format's url meta field (#20242)
+* [utils] Strip #HttpOnly_ prefix from cookies files (#20219)
+
+Extractors
+* [francetv:site] Relax video id regular expression (#20268)
+* [toutv] Detect invalid login error
+* [toutv] Fix authentication (#20261)
++ [urplay] Extract timestamp (#20235)
++ [openload] Add support for oload.space (#20246)
+* [facebook] Improve uploader extraction (#20250)
+* [bbc] Use compat_etree_Element
+* [crunchyroll] Use compat_etree_Element
+* [npo] Improve ISM extraction
+* [rai] Improve extraction (#20253)
+* [paramountnetwork] Fix mgid extraction (#20241)
+* [libsyn] Improve extraction (#20229)
++ [youtube] Add more invidious instances to URL regular expression (#20228)
+* [spankbang] Fix extraction (#20023)
+* [espn] Extend URL regular expression (#20013)
+* [sixplay] Handle videos with empty assets (#20016)
++ [vimeo] Add support for Vimeo Pro portfolio protected videos (#20070)
+
+
 version 2019.03.01
 
 Core

From 10734553feef497e2810a23bbe62a0b3d630e78d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Mar 2019 02:53:18 +0700
Subject: [PATCH 468/510] release 2019.03.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 71a500f04..5f97e2cbe 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.03.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.09*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.03.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.03.01
+[debug] youtube-dl version 2019.03.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 272191a01..eda94ad33 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.03.09
 
 Core
 * [extractor/common] Use compat_etree_Element
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 42ba37f15..f72fee57f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.03.01'
+__version__ = '2019.03.09'

From 0d08bcdb70008f0d500afbd19059b3c0971a4776 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 10 Mar 2019 09:37:28 +0100
Subject: [PATCH 469/510] [fox] detect geo restriction and authentication
 errors(#20208)

---
 youtube_dl/extractor/fox.py | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 0ffceeb7c..f30d3cba8 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -6,10 +6,12 @@ import uuid
 
 from .adobepass import AdobePassIE
 from ..compat import (
+    compat_HTTPError,
     compat_str,
     compat_urllib_parse_unquote,
 )
 from ..utils import (
+    ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_duration,
@@ -48,6 +50,7 @@ class FOXIE(AdobePassIE):
         'url': 'https://www.fox.com/watch/30056b295fb57f7452aeeb4920bc3024/',
         'only_matching': True,
     }]
+    _GEO_BYPASS = False
     _HOME_PAGE_URL = 'https://www.fox.com/'
     _API_KEY = 'abdcbed02c124d393b39e818a4312055'
     _access_token = None
@@ -58,9 +61,22 @@ class FOXIE(AdobePassIE):
         }
         if self._access_token:
             headers['Authorization'] = 'Bearer ' + self._access_token
-        return self._download_json(
-            'https://api2.fox.com/v2.0/' + path,
-            video_id, data=data, headers=headers)
+        try:
+            return self._download_json(
+                'https://api2.fox.com/v2.0/' + path,
+                video_id, data=data, headers=headers)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.status == 403:
+                entitlement_issues = self._parse_json(
+                    e.cause.read().decode(), video_id)['entitlementIssues']
+                for e in entitlement_issues:
+                    if e.get('errorCode') == 1005:
+                        raise ExtractorError(
+                            'This video is only available via cable service provider '
+                            'subscription. You may want to use --cookies.', expected=True)
+                messages = ', '.join([e['message'] for e in entitlement_issues])
+                raise ExtractorError(messages, expected=True)
+            raise
 
     def _real_initialize(self):
         if not self._access_token:
@@ -81,7 +97,15 @@ class FOXIE(AdobePassIE):
 
         title = video['name']
         release_url = video['url']
-        m3u8_url = self._download_json(release_url, video_id)['playURL']
+        try:
+            m3u8_url = self._download_json(release_url, video_id)['playURL']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.status == 403:
+                error = self._parse_json(e.cause.read().decode(), video_id)
+                if error.get('exception') == 'GeoLocationBlocked':
+                    self.raise_geo_restricted(countries=['US'])
+                raise ExtractorError(error['description'], expected=True)
+            raise
         formats = self._extract_m3u8_formats(
             m3u8_url, video_id, 'mp4',
             entry_protocol='m3u8_native', m3u8_id='hls')

From 276550371313dbfe7d94ceb294bd1284c1e7c404 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 10 Mar 2019 15:03:32 +0100
Subject: [PATCH 470/510] [vimeo:review] improve config url extraction and
 extract original format(closes #20305)

---
 youtube_dl/extractor/vimeo.py | 64 +++++++++++++++++++++--------------
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6f32ea6f1..e3ec550f0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -195,6 +195,32 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'subtitles': subtitles,
         }
 
+    def _extract_original_format(self, url, video_id):
+        download_data = self._download_json(
+            url, video_id, fatal=False,
+            query={'action': 'load_download_config'},
+            headers={'X-Requested-With': 'XMLHttpRequest'})
+        if download_data:
+            source_file = download_data.get('source_file')
+            if isinstance(source_file, dict):
+                download_url = source_file.get('download_url')
+                if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
+                    source_name = source_file.get('public_name', 'Original')
+                    if self._is_valid_url(download_url, video_id, '%s video' % source_name):
+                        ext = (try_get(
+                            source_file, lambda x: x['extension'],
+                            compat_str) or determine_ext(
+                            download_url, None) or 'mp4').lower()
+                        return {
+                            'url': download_url,
+                            'ext': ext,
+                            'width': int_or_none(source_file.get('width')),
+                            'height': int_or_none(source_file.get('height')),
+                            'filesize': parse_filesize(source_file.get('size')),
+                            'format_id': source_name,
+                            'preference': 1,
+                        }
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -659,29 +685,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
             comment_count = None
 
         formats = []
-        download_request = sanitized_Request('https://vimeo.com/%s?action=load_download_config' % video_id, headers={
-            'X-Requested-With': 'XMLHttpRequest'})
-        download_data = self._download_json(download_request, video_id, fatal=False)
-        if download_data:
-            source_file = download_data.get('source_file')
-            if isinstance(source_file, dict):
-                download_url = source_file.get('download_url')
-                if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
-                    source_name = source_file.get('public_name', 'Original')
-                    if self._is_valid_url(download_url, video_id, '%s video' % source_name):
-                        ext = (try_get(
-                            source_file, lambda x: x['extension'],
-                            compat_str) or determine_ext(
-                            download_url, None) or 'mp4').lower()
-                        formats.append({
-                            'url': download_url,
-                            'ext': ext,
-                            'width': int_or_none(source_file.get('width')),
-                            'height': int_or_none(source_file.get('height')),
-                            'filesize': parse_filesize(source_file.get('size')),
-                            'format_id': source_name,
-                            'preference': 1,
-                        })
+
+        source_format = self._extract_original_format(
+            'https://vimeo.com/' + video_id, video_id)
+        if source_format:
+            formats.append(source_format)
 
         info_dict_config = self._parse_config(config, video_id)
         formats.extend(info_dict_config['formats'])
@@ -940,7 +948,7 @@ class VimeoGroupsIE(VimeoAlbumIE):
 class VimeoReviewIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:review'
     IE_DESC = 'Review pages on vimeo'
-    _VALID_URL = r'https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
+    _VALID_URL = r'(?P<url>https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)/[0-9a-f]{10})'
     _TESTS = [{
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
@@ -992,7 +1000,8 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
             data = self._parse_json(self._search_regex(
                 r'window\s*=\s*_extend\(window,\s*({.+?})\);', webpage, 'data',
                 default=NO_DEFAULT if video_password_verified else '{}'), video_id)
-            config_url = data.get('vimeo_esi', {}).get('config', {}).get('configUrl')
+            config = data.get('vimeo_esi', {}).get('config', {})
+            config_url = config.get('configUrl') or try_get(config, lambda x: x['clipData']['configUrl'])
         if config_url is None:
             self._verify_video_password(webpage_url, video_id, webpage)
             config_url = self._get_config_url(
@@ -1000,10 +1009,13 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
         return config_url
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        page_url, video_id = re.match(self._VALID_URL, url).groups()
         config_url = self._get_config_url(url, video_id)
         config = self._download_json(config_url, video_id)
         info_dict = self._parse_config(config, video_id)
+        source_format = self._extract_original_format(page_url, video_id)
+        if source_format:
+            info_dict['formats'].append(source_format)
         self._vimeo_sort_formats(info_dict['formats'])
         info_dict['id'] = video_id
         return info_dict

From 067aa17edf5a46a8cbc4d6b90864eddf051fa2bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Mar 2019 19:14:41 +0700
Subject: [PATCH 471/510] Start moving to ytdl-org

---
 .github/ISSUE_TEMPLATE.md               | 10 +++---
 .github/ISSUE_TEMPLATE_tmpl.md          | 10 +++---
 .github/PULL_REQUEST_TEMPLATE.md        |  4 +--
 CONTRIBUTING.md                         | 20 +++++------
 README.md                               | 42 +++++++++++------------
 devscripts/buildserver.py               |  2 +-
 devscripts/create-github-release.py     |  4 +--
 devscripts/gh-pages/update-feed.py      |  4 +--
 devscripts/release.sh                   |  2 +-
 devscripts/show-downloads-statistics.py |  2 +-
 setup.py                                |  2 +-
 test/test_InfoExtractor.py              | 16 ++++-----
 test/test_YoutubeDL.py                  |  6 ++--
 test/test_all_urls.py                   |  6 ++--
 youtube-dl.plugin.zsh                   |  2 +-
 youtube_dl/YoutubeDL.py                 | 16 ++++-----
 youtube_dl/__init__.py                  |  2 +-
 youtube_dl/compat.py                    |  6 ++--
 youtube_dl/downloader/external.py       |  4 +--
 youtube_dl/downloader/f4m.py            |  6 ++--
 youtube_dl/downloader/hls.py            |  4 +--
 youtube_dl/downloader/http.py           |  2 +-
 youtube_dl/extractor/arkena.py          |  2 +-
 youtube_dl/extractor/bambuser.py        |  4 +--
 youtube_dl/extractor/bbc.py             |  2 +-
 youtube_dl/extractor/brightcove.py      |  6 ++--
 youtube_dl/extractor/ceskatelevize.py   |  2 +-
 youtube_dl/extractor/common.py          | 20 +++++------
 youtube_dl/extractor/commonmistakes.py  |  2 +-
 youtube_dl/extractor/crunchyroll.py     |  4 +--
 youtube_dl/extractor/dailymotion.py     |  4 +--
 youtube_dl/extractor/dreisat.py         |  2 +-
 youtube_dl/extractor/francetv.py        |  2 +-
 youtube_dl/extractor/generic.py         | 10 +++---
 youtube_dl/extractor/googledrive.py     |  2 +-
 youtube_dl/extractor/kuwo.py            |  2 +-
 youtube_dl/extractor/liveleak.py        |  6 ++--
 youtube_dl/extractor/msn.py             |  2 +-
 youtube_dl/extractor/nhl.py             |  2 +-
 youtube_dl/extractor/noco.py            |  2 +-
 youtube_dl/extractor/once.py            |  2 +-
 youtube_dl/extractor/pbs.py             |  6 ++--
 youtube_dl/extractor/pluralsight.py     |  6 ++--
 youtube_dl/extractor/pornhub.py         |  2 +-
 youtube_dl/extractor/prosiebensat1.py   |  4 +--
 youtube_dl/extractor/radiocanada.py     |  2 +-
 youtube_dl/extractor/rtlnl.py           |  2 +-
 youtube_dl/extractor/theplatform.py     |  4 +--
 youtube_dl/extractor/toutv.py           |  2 +-
 youtube_dl/extractor/udemy.py           |  2 +-
 youtube_dl/extractor/ustream.py         |  2 +-
 youtube_dl/extractor/veehd.py           |  2 +-
 youtube_dl/extractor/vevo.py            |  2 +-
 youtube_dl/extractor/vimeo.py           |  4 +--
 youtube_dl/extractor/vk.py              |  2 +-
 youtube_dl/extractor/vlive.py           |  2 +-
 youtube_dl/extractor/yandexmusic.py     |  4 +--
 youtube_dl/extractor/youtube.py         | 44 ++++++++++++-------------
 youtube_dl/options.py                   |  2 +-
 youtube_dl/update.py                    |  2 +-
 youtube_dl/utils.py                     | 18 +++++-----
 61 files changed, 182 insertions(+), 182 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5f97e2cbe..911e912a4 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,12 +6,12 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.09*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.09*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
 - [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.03.09**
 
 ### Before submitting an *issue* make sure you have:
-- [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
-- [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
+- [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections
+- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 - [ ] Checked that provided video/audio/playlist URLs (if any) are alive and playable in a browser
 
 ### What is the purpose of your *issue*?
@@ -51,11 +51,11 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 - Single video: https://youtu.be/BaW_jenozKc
 - Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
 
-Note that **youtube-dl does not support sites dedicated to [copyright infringement](https://github.com/rg3/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
+Note that **youtube-dl does not support sites dedicated to [copyright infringement](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 
 ---
 
 ### Description of your *issue*, suggested solution and other information
 
-Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
+Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
 If work on your *issue* requires account credentials please provide them or explain how one can obtain them.
diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md
index 8edbd5a0f..8b7e73417 100644
--- a/.github/ISSUE_TEMPLATE_tmpl.md
+++ b/.github/ISSUE_TEMPLATE_tmpl.md
@@ -6,12 +6,12 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
 - [ ] I've **verified** and **I assure** that I'm running youtube-dl **%(version)s**
 
 ### Before submitting an *issue* make sure you have:
-- [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
-- [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
+- [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections
+- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 - [ ] Checked that provided video/audio/playlist URLs (if any) are alive and playable in a browser
 
 ### What is the purpose of your *issue*?
@@ -51,11 +51,11 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 - Single video: https://youtu.be/BaW_jenozKc
 - Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
 
-Note that **youtube-dl does not support sites dedicated to [copyright infringement](https://github.com/rg3/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
+Note that **youtube-dl does not support sites dedicated to [copyright infringement](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 
 ---
 
 ### Description of your *issue*, suggested solution and other information
 
-Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
+Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
 If work on your *issue* requires account credentials please provide them or explain how one can obtain them.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index ba4ca7553..e69b907d8 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -7,8 +7,8 @@
 ---
 
 ### Before submitting a *pull request* make sure you have:
-- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/rg3/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/rg3/youtube-dl#youtube-dl-coding-conventions) sections
-- [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
+- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) sections
+- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
 - [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)
 
 ### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6c1739860..cd9ccbe96 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -42,11 +42,11 @@ Before reporting any issue, type `youtube-dl -U`. This should report that you're
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/rg3/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
 
 ###  Why are existing options not enough?
 
-Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
+Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
 
 ###  Is there enough context in your bug report?
 
@@ -70,7 +70,7 @@ It may sound strange, but some bug reports we receive are completely unrelated t
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build youtube-dl and can [download the builds](https://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build youtube-dl and can [download the builds](https://ytdl-org.github.io/youtube-dl/download.html) or get them from their distribution.
 
 To run youtube-dl as a developer, you don't need to build anything either. Simply execute
 
@@ -98,7 +98,7 @@ If you want to add support for a new site, first of all **make sure** this site
 
 After you have ensured this site is distributing its content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
-1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
+1. [Fork this repository](https://github.com/ytdl-org/youtube-dl/fork)
 2. Check out the source code with:
 
         git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git
@@ -150,9 +150,9 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](http://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
@@ -177,7 +177,7 @@ Extractors are very fragile by nature since they depend on the layout of the sou
 
 ### Mandatory and optional metafields
 
-For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
 
  - `id` (media identifier)
  - `title` (media title)
@@ -185,7 +185,7 @@ For extraction to work youtube-dl relies on metadata your extractor extracts and
 
 In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` as mandatory. Thus the aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken.
 
-[Any field](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+[Any field](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
 #### Example
 
@@ -341,7 +341,7 @@ Incorrect:
 
 ### Use convenience conversion and parsing functions
 
-Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
 
 Use `url_or_none` for safe URL processing.
 
@@ -349,7 +349,7 @@ Use `try_get` for safe metadata extraction from parsed JSON.
 
 Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. 
 
-Explore [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
+Explore [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
 
 #### More examples
 
diff --git a/README.md b/README.md
index c1572f771..e476045b2 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/rg3/youtube-dl.svg?branch=master)](https://travis-ci.org/rg3/youtube-dl)
+[![Build Status](https://travis-ci.org/ytdl-org/youtube-dl.svg?branch=master)](https://travis-ci.org/ytdl-org/youtube-dl)
 
 youtube-dl - download videos from youtube.com or other video platforms
 
@@ -43,7 +43,7 @@ Or with [MacPorts](https://www.macports.org/):
 
     sudo port install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://ytdl-org.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
 **youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
@@ -685,7 +685,7 @@ You can merge the video and audio of two formats into a single file using `-f <v
 
 Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26, youtube-dl uses `-f bestvideo+bestaudio/best` as the default format selection (see [#5447](https://github.com/rg3/youtube-dl/issues/5447), [#5456](https://github.com/rg3/youtube-dl/issues/5456)). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26, youtube-dl uses `-f bestvideo+bestaudio/best` as the default format selection (see [#5447](https://github.com/ytdl-org/youtube-dl/issues/5447), [#5456](https://github.com/ytdl-org/youtube-dl/issues/5456)). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
@@ -736,7 +736,7 @@ $ youtube-dl --dateafter 20000101 --datebefore 20091231
 
 ### How do I update youtube-dl?
 
-If you've followed [our manual installation instructions](https://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
+If you've followed [our manual installation instructions](https://ytdl-org.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
@@ -746,7 +746,7 @@ As a last resort, you can also uninstall the version installed by your package m
 
     sudo apt-get remove -y youtube-dl
 
-Afterwards, simply follow [our manual installation instructions](https://rg3.github.io/youtube-dl/download.html):
+Afterwards, simply follow [our manual installation instructions](https://ytdl-org.github.io/youtube-dl/download.html):
 
 ```
 sudo wget https://yt-dl.org/latest/youtube-dl -O /usr/local/bin/youtube-dl
@@ -780,7 +780,7 @@ Most people asking this question are not aware that youtube-dl now defaults to d
 
 ### I get HTTP error 402 when trying to download a video. What's this?
 
-Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a web browser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
+Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/ytdl-org/youtube-dl/issues/154), but at the moment, your best course of action is pointing a web browser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
 
 ### Do I need any other programs?
 
@@ -845,7 +845,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 ([#342](https://github.com/rg3/youtube-dl/issues/342)) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 ([#342](https://github.com/ytdl-org/youtube-dl/issues/342)) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws an error due to missing `MSVCR100.dll`
 
@@ -904,7 +904,7 @@ When youtube-dl detects an HLS video, it can download it either with the built-i
 
 When youtube-dl knows that one particular downloader works better for a given website, that downloader will be picked. Otherwise, youtube-dl will pick the best downloader for general compatibility, which at the moment happens to be ffmpeg. This choice may change in future versions of youtube-dl, with improvements of the built-in downloader and/or ffmpeg.
 
-In particular, the generic extractor (used when your website is not in the [list of supported sites by youtube-dl](https://rg3.github.io/youtube-dl/supportedsites.html) cannot mandate one specific downloader.
+In particular, the generic extractor (used when your website is not in the [list of supported sites by youtube-dl](https://ytdl-org.github.io/youtube-dl/supportedsites.html) cannot mandate one specific downloader.
 
 If you put either `--hls-prefer-native` or `--hls-prefer-ffmpeg` into your configuration, a different subset of videos will fail to download correctly. Instead, it is much better to [file an issue](https://yt-dl.org/bug) or a pull request which details why the native or the ffmpeg HLS downloader is a better choice for your use case.
 
@@ -944,7 +944,7 @@ youtube-dl is an open-source project manned by too few volunteers, so we'd rathe
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build youtube-dl and can [download the builds](https://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build youtube-dl and can [download the builds](https://ytdl-org.github.io/youtube-dl/download.html) or get them from their distribution.
 
 To run youtube-dl as a developer, you don't need to build anything either. Simply execute
 
@@ -972,7 +972,7 @@ If you want to add support for a new site, first of all **make sure** this site
 
 After you have ensured this site is distributing its content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
-1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
+1. [Fork this repository](https://github.com/ytdl-org/youtube-dl/fork)
 2. Check out the source code with:
 
         git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git
@@ -1024,9 +1024,9 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](http://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
@@ -1051,7 +1051,7 @@ Extractors are very fragile by nature since they depend on the layout of the sou
 
 ### Mandatory and optional metafields
 
-For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl:
 
  - `id` (media identifier)
  - `title` (media title)
@@ -1059,7 +1059,7 @@ For extraction to work youtube-dl relies on metadata your extractor extracts and
 
 In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` as mandatory. Thus the aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken.
 
-[Any field](https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+[Any field](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
 #### Example
 
@@ -1215,7 +1215,7 @@ Incorrect:
 
 ### Use convenience conversion and parsing functions
 
-Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
 
 Use `url_or_none` for safe URL processing.
 
@@ -1223,7 +1223,7 @@ Use `try_get` for safe metadata extraction from parsed JSON.
 
 Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. 
 
-Explore [`youtube_dl/utils.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
+Explore [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
 
 #### More examples
 
@@ -1242,7 +1242,7 @@ view_count = int_or_none(video.get('views'))
 
 # EMBEDDING YOUTUBE-DL
 
-youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/ytdl-org/youtube-dl/issues/new).
 
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
@@ -1255,7 +1255,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/ytdl-org/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
@@ -1296,7 +1296,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
+Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
@@ -1342,11 +1342,11 @@ Before reporting any issue, type `youtube-dl -U`. This should report that you're
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/rg3/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
 
 ###  Why are existing options not enough?
 
-Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
+Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
 
 ###  Is there enough context in your bug report?
 
diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index 1344b4d87..4a4295ba9 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -322,7 +322,7 @@ class GITBuilder(GITInfoBuilder):
 
 
 class YoutubeDLBuilder(object):
-    authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile']
+    authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org']
 
     def __init__(self, **kwargs):
         if self.repoName != 'youtube-dl':
diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py
index 30716ad8e..428111b3f 100644
--- a/devscripts/create-github-release.py
+++ b/devscripts/create-github-release.py
@@ -27,8 +27,8 @@ from youtube_dl.utils import (
 
 
 class GitHubReleaser(object):
-    _API_URL = 'https://api.github.com/repos/rg3/youtube-dl/releases'
-    _UPLOADS_URL = 'https://uploads.github.com/repos/rg3/youtube-dl/releases/%s/assets?name=%s'
+    _API_URL = 'https://api.github.com/repos/ytdl-org/youtube-dl/releases'
+    _UPLOADS_URL = 'https://uploads.github.com/repos/ytdl-org/youtube-dl/releases/%s/assets?name=%s'
     _NETRC_MACHINE = 'github.com'
 
     def __init__(self, debuglevel=0):
diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py
index e93eb60fb..506a62377 100755
--- a/devscripts/gh-pages/update-feed.py
+++ b/devscripts/gh-pages/update-feed.py
@@ -10,7 +10,7 @@ import textwrap
 atom_template = textwrap.dedent("""\
     <?xml version="1.0" encoding="utf-8"?>
     <feed xmlns="http://www.w3.org/2005/Atom">
-        <link rel="self" href="http://rg3.github.io/youtube-dl/update/releases.atom" />
+        <link rel="self" href="http://ytdl-org.github.io/youtube-dl/update/releases.atom" />
         <title>youtube-dl releases</title>
         <id>https://yt-dl.org/feed/youtube-dl-updates-feed</id>
         <updated>@TIMESTAMP@</updated>
@@ -21,7 +21,7 @@ entry_template = textwrap.dedent("""
     <entry>
         <id>https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@</id>
         <title>New version @VERSION@</title>
-        <link href="http://rg3.github.io/youtube-dl" />
+        <link href="http://ytdl-org.github.io/youtube-dl" />
         <content type="xhtml">
             <div xmlns="http://www.w3.org/1999/xhtml">
                 Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a>
diff --git a/devscripts/release.sh b/devscripts/release.sh
index 4db5def5d..4c413bf6d 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -96,7 +96,7 @@ git push origin "$version"
 REV=$(git rev-parse HEAD)
 make youtube-dl youtube-dl.tar.gz
 read -p "VM running? (y/n) " -n 1
-wget "http://$buildserver/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
+wget "http://$buildserver/build/ytdl-org/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
 mkdir -p "build/$version"
 mv youtube-dl youtube-dl.exe "build/$version"
 mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"
diff --git a/devscripts/show-downloads-statistics.py b/devscripts/show-downloads-statistics.py
index e25d28411..6c8d1cc2d 100644
--- a/devscripts/show-downloads-statistics.py
+++ b/devscripts/show-downloads-statistics.py
@@ -24,7 +24,7 @@ total_bytes = 0
 
 for page in itertools.count(1):
     releases = json.loads(compat_urllib_request.urlopen(
-        'https://api.github.com/repos/rg3/youtube-dl/releases?page=%s' % page
+        'https://api.github.com/repos/ytdl-org/youtube-dl/releases?page=%s' % page
     ).read().decode('utf-8'))
 
     if not releases:
diff --git a/setup.py b/setup.py
index dfb669ad2..af68b485e 100644
--- a/setup.py
+++ b/setup.py
@@ -104,7 +104,7 @@ setup(
     version=__version__,
     description=DESCRIPTION,
     long_description=LONG_DESCRIPTION,
-    url='https://github.com/rg3/youtube-dl',
+    url='https://github.com/ytdl-org/youtube-dl',
     author='Ricardo Garcia',
     author_email='ytdl@yt-dl.org',
     maintainer='Sergey M.',
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index f0aa8466b..da6cd39b6 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -201,7 +201,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
     def test_parse_m3u8_formats(self):
         _TEST_CASES = [
             (
-                # https://github.com/rg3/youtube-dl/issues/11507
+                # https://github.com/ytdl-org/youtube-dl/issues/11507
                 # http://pluzz.francetv.fr/videos/le_ministere.html
                 'pluzz_francetv_11507',
                 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
@@ -263,7 +263,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 }]
             ),
             (
-                # https://github.com/rg3/youtube-dl/issues/11995
+                # https://github.com/ytdl-org/youtube-dl/issues/11995
                 # http://teamcoco.com/video/clueless-gamer-super-bowl-for-honor
                 'teamcoco_11995',
                 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
@@ -337,7 +337,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 }]
             ),
             (
-                # https://github.com/rg3/youtube-dl/issues/12211
+                # https://github.com/ytdl-org/youtube-dl/issues/12211
                 # http://video.toggle.sg/en/series/whoopie-s-world/ep3/478601
                 'toggle_mobile_12211',
                 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
@@ -501,7 +501,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 }]
             ),
             (
-                # https://github.com/rg3/youtube-dl/issues/18923
+                # https://github.com/ytdl-org/youtube-dl/issues/18923
                 # https://www.ted.com/talks/boris_hesser_a_grassroots_healthcare_revolution_in_africa
                 'ted_18923',
                 'http://hls.ted.com/talks/31241.m3u8',
@@ -570,9 +570,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
     def test_parse_mpd_formats(self):
         _TEST_CASES = [
             (
-                # https://github.com/rg3/youtube-dl/issues/13919
+                # https://github.com/ytdl-org/youtube-dl/issues/13919
                 # Also tests duplicate representation ids, see
-                # https://github.com/rg3/youtube-dl/issues/15111
+                # https://github.com/ytdl-org/youtube-dl/issues/15111
                 'float_duration',
                 'http://unknown/manifest.mpd',
                 [{
@@ -652,7 +652,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 1080,
                 }]
             ), (
-                # https://github.com/rg3/youtube-dl/pull/14844
+                # https://github.com/ytdl-org/youtube-dl/pull/14844
                 'urls_only',
                 'http://unknown/manifest.mpd',
                 [{
@@ -748,7 +748,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
     def test_parse_f4m_formats(self):
         _TEST_CASES = [
             (
-                # https://github.com/rg3/youtube-dl/issues/14660
+                # https://github.com/ytdl-org/youtube-dl/issues/14660
                 'custom_base_url',
                 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
                 [{
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 1d7452744..ce9666171 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -411,7 +411,7 @@ class TestFormatSelection(unittest.TestCase):
         # For extractors with incomplete formats (all formats are audio-only or
         # video-only) best and worst should fallback to corresponding best/worst
         # video-only or audio-only formats (as per
-        # https://github.com/rg3/youtube-dl/pull/5556)
+        # https://github.com/ytdl-org/youtube-dl/pull/5556)
         formats = [
             {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
             {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
@@ -442,7 +442,7 @@ class TestFormatSelection(unittest.TestCase):
         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
 
     def test_format_selection_issue_10083(self):
-        # See https://github.com/rg3/youtube-dl/issues/10083
+        # See https://github.com/ytdl-org/youtube-dl/issues/10083
         formats = [
             {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
             {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
@@ -853,7 +853,7 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(result, [2, 3, 4])
 
     def test_urlopen_no_file_protocol(self):
-        # see https://github.com/rg3/youtube-dl/issues/8227
+        # see https://github.com/ytdl-org/youtube-dl/issues/8227
         ydl = YDL()
         self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
 
diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index cd1cd4b24..465ce0050 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -110,7 +110,7 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('https://vimeo.com/user7108434/videos', ['vimeo:user'])
         self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])
 
-    # https://github.com/rg3/youtube-dl/issues/1930
+    # https://github.com/ytdl-org/youtube-dl/issues/1930
     def test_soundcloud_not_matching_sets(self):
         self.assertMatch('http://soundcloud.com/floex/sets/gone-ep', ['soundcloud:set'])
 
@@ -119,12 +119,12 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('http://tatianamaslanydaily.tumblr.com/post/54196191430', ['Tumblr'])
 
     def test_pbs(self):
-        # https://github.com/rg3/youtube-dl/issues/2350
+        # https://github.com/ytdl-org/youtube-dl/issues/2350
         self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['pbs'])
         self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['pbs'])
 
     def test_yahoo_https(self):
-        # https://github.com/rg3/youtube-dl/issues/2701
+        # https://github.com/ytdl-org/youtube-dl/issues/2701
         self.assertMatch(
             'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
             ['Yahoo'])
diff --git a/youtube-dl.plugin.zsh b/youtube-dl.plugin.zsh
index 4edab5214..17ab1341a 100644
--- a/youtube-dl.plugin.zsh
+++ b/youtube-dl.plugin.zsh
@@ -7,7 +7,7 @@
 # https://github.com/zsh-users/antigen
 
 # Install youtube-dl:
-# antigen bundle rg3/youtube-dl
+# antigen bundle ytdl-org/youtube-dl
 # Bundles installed by antigen are available for use immediately.
 
 # Update youtube-dl (and all other antigen bundles):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bc9fc270c..3b92acd97 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -889,7 +889,7 @@ class YoutubeDL(object):
             # url_transparent. In such cases outer metadata (from ie_result)
             # should be propagated to inner one (info). For this to happen
             # _type of info should be overridden with url_transparent. This
-            # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
+            # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
             if new_result.get('_type') == 'url':
                 new_result['_type'] = 'url_transparent'
 
@@ -1606,7 +1606,7 @@ class YoutubeDL(object):
         # by extractor are incomplete or not (i.e. whether extractor provides only
         # video-only or audio-only formats) for proper formats selection for
         # extractors with such incomplete formats (see
-        # https://github.com/rg3/youtube-dl/pull/5556).
+        # https://github.com/ytdl-org/youtube-dl/pull/5556).
         # Since formats may be filtered during format selection and may not match
         # the original formats the results may be incorrect. Thus original formats
         # or pre-calculated metrics should be passed to format selection routines
@@ -1614,7 +1614,7 @@ class YoutubeDL(object):
         # We will pass a context object containing all necessary additional data
         # instead of just formats.
         # This fixes incorrect format selection issue (see
-        # https://github.com/rg3/youtube-dl/issues/10083).
+        # https://github.com/ytdl-org/youtube-dl/issues/10083).
         incomplete_formats = (
             # All formats are video-only or
             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
@@ -1810,7 +1810,7 @@ class YoutubeDL(object):
                     if sub_info.get('data') is not None:
                         try:
                             # Use newline='' to prevent conversion of newline characters
-                            # See https://github.com/rg3/youtube-dl/issues/10268
+                            # See https://github.com/ytdl-org/youtube-dl/issues/10268
                             with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                                 subfile.write(sub_info['data'])
                         except (OSError, IOError):
@@ -2229,7 +2229,7 @@ class YoutubeDL(object):
             return
 
         if type('') is not compat_str:
-            # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
+            # Python 2.6 on SLES11 SP1 (https://github.com/ytdl-org/youtube-dl/issues/3326)
             self.report_warning(
                 'Your Python is broken! Update to a newer and supported version')
 
@@ -2323,7 +2323,7 @@ class YoutubeDL(object):
                 proxies = {'http': opts_proxy, 'https': opts_proxy}
         else:
             proxies = compat_urllib_request.getproxies()
-            # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
+            # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']
         proxy_handler = PerRequestProxyHandler(proxies)
@@ -2336,7 +2336,7 @@ class YoutubeDL(object):
         # When passing our own FileHandler instance, build_opener won't add the
         # default FileHandler and allows us to disable the file protocol, which
         # can be used for malicious purposes (see
-        # https://github.com/rg3/youtube-dl/issues/8227)
+        # https://github.com/ytdl-org/youtube-dl/issues/8227)
         file_handler = compat_urllib_request.FileHandler()
 
         def file_open(*args, **kwargs):
@@ -2348,7 +2348,7 @@ class YoutubeDL(object):
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play
-        # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
+        # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
         opener.addheaders = []
         self._opener = opener
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ba435ea42..94788d936 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -48,7 +48,7 @@ from .YoutubeDL import YoutubeDL
 def _real_main(argv=None):
     # Compatibility fixes for Windows
     if sys.platform == 'win32':
-        # https://github.com/rg3/youtube-dl/issues/820
+        # https://github.com/ytdl-org/youtube-dl/issues/820
         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 
     workaround_optparse_bug9161()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b2fe62f12..7992a23ca 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2364,7 +2364,7 @@ except ImportError:  # Python 2
 
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
-    # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
+    # is apparently broken (see https://github.com/ytdl-org/youtube-dl/pull/6244)
 
     def compat_urllib_parse_unquote_to_bytes(string):
         """unquote_to_bytes('abc%20def') -> b'abc def'."""
@@ -2828,7 +2828,7 @@ else:
     compat_socket_create_connection = socket.create_connection
 
 
-# Fix https://github.com/rg3/youtube-dl/issues/4223
+# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
     op = optparse.OptionParser()
@@ -2953,7 +2953,7 @@ if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4,
     # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
     # names, see the original PyPy issue [1] and the youtube-dl one [2].
     # 1. https://bitbucket.org/pypy/pypy/issues/2360/windows-ctypescdll-typeerror-function-name
-    # 2. https://github.com/rg3/youtube-dl/pull/4392
+    # 2. https://github.com/ytdl-org/youtube-dl/pull/4392
     def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
         real = ctypes.WINFUNCTYPE(*args, **kwargs)
 
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 22e6093b3..5f73f7f0f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -239,7 +239,7 @@ class FFmpegFD(ExternalFD):
             # setting -seekable prevents ffmpeg from guessing if the server
             # supports seeking(by adding the header `Range: bytes=0-`), which
             # can cause problems in some cases
-            # https://github.com/rg3/youtube-dl/issues/11800#issuecomment-275037127
+            # https://github.com/ytdl-org/youtube-dl/issues/11800#issuecomment-275037127
             # http://trac.ffmpeg.org/ticket/6125#comment:10
             args += ['-seekable', '1' if seekable else '0']
 
@@ -334,7 +334,7 @@ class FFmpegFD(ExternalFD):
             # mp4 file couldn't be played, but if we ask ffmpeg to quit it
             # produces a file that is playable (this is mostly useful for live
             # streams). Note that Windows is not affected and produces playable
-            # files (see https://github.com/rg3/youtube-dl/issues/8300).
+            # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
             if sys.platform != 'win32':
                 proc.communicate(b'q')
             raise
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 15e71be9a..9b15a0e15 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -324,8 +324,8 @@ class F4mFD(FragmentFD):
         urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
         man_url = urlh.geturl()
         # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
-        # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
-        # and https://github.com/rg3/youtube-dl/issues/7823)
+        # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
+        # and https://github.com/ytdl-org/youtube-dl/issues/7823)
         manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()
 
         doc = compat_etree_fromstring(manifest)
@@ -409,7 +409,7 @@ class F4mFD(FragmentFD):
                             # In tests, segments may be truncated, and thus
                             # FlvReader may not be able to parse the whole
                             # chunk. If so, write the segment as is
-                            # See https://github.com/rg3/youtube-dl/issues/9214
+                            # See https://github.com/ytdl-org/youtube-dl/issues/9214
                             dest_stream.write(down_data)
                             break
                         raise
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 4def8e2d5..419e73576 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -152,8 +152,8 @@ class HlsFD(FragmentFD):
                         except compat_urllib_error.HTTPError as err:
                             # Unavailable (possibly temporary) fragments may be served.
                             # First we try to retry then either skip or abort.
-                            # See https://github.com/rg3/youtube-dl/issues/10165,
-                            # https://github.com/rg3/youtube-dl/issues/10448).
+                            # See https://github.com/ytdl-org/youtube-dl/issues/10165,
+                            # https://github.com/ytdl-org/youtube-dl/issues/10448).
                             count += 1
                             if count <= fragment_retries:
                                 self.report_retry_fragment(err, frag_index, count, fragment_retries)
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 5b1e96013..08670ee3c 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -111,7 +111,7 @@ class HttpFD(FileDownloader):
                 # to match the value of requested Range HTTP header. This is due to a webservers
                 # that don't support resuming and serve a whole file with no Content-Range
                 # set in response despite of requested Range (see
-                # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
+                # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
                 if has_range:
                     content_range = ctx.data.headers.get('Content-Range')
                     if content_range:
diff --git a/youtube_dl/extractor/arkena.py b/youtube_dl/extractor/arkena.py
index 4495ddbb0..854f58767 100644
--- a/youtube_dl/extractor/arkena.py
+++ b/youtube_dl/extractor/arkena.py
@@ -103,7 +103,7 @@ class ArkenaIE(InfoExtractor):
                         f_url, video_id, mpd_id=kind, fatal=False))
                 elif kind == 'silverlight':
                     # TODO: process when ism is supported (see
-                    # https://github.com/rg3/youtube-dl/issues/8118)
+                    # https://github.com/ytdl-org/youtube-dl/issues/8118)
                     continue
                 else:
                     tbr = float_or_none(f.get('Bitrate'), 1000)
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 34f1b3d83..4400ff9c1 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -23,7 +23,7 @@ class BambuserIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://bambuser.com/v/4050584',
-        # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
+        # MD5 seems to be flaky, see https://travis-ci.org/ytdl-org/youtube-dl/jobs/14051016#L388
         # 'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
         'info_dict': {
             'id': '4050584',
@@ -38,7 +38,7 @@ class BambuserIE(InfoExtractor):
         },
         'params': {
             # It doesn't respect the 'Range' header, it would download the whole video
-            # caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59
+            # caused the travis builds to fail: https://travis-ci.org/ytdl-org/youtube-dl/jobs/14493845#L59
             'skip_download': True,
         },
     }
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index d479d2577..e76507951 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -208,7 +208,7 @@ class BBCCoUkIE(InfoExtractor):
             },
             'skip': 'Now it\'s really geo-restricted',
         }, {
-            # compact player (https://github.com/rg3/youtube-dl/issues/8147)
+            # compact player (https://github.com/ytdl-org/youtube-dl/issues/8147)
             'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
             'info_dict': {
                 'id': 'p028bfkj',
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 465ae396e..c0345e2c3 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -126,7 +126,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             'playlist_mincount': 7,
         },
         {
-            # playlist with 'playlistTab' (https://github.com/rg3/youtube-dl/issues/9965)
+            # playlist with 'playlistTab' (https://github.com/ytdl-org/youtube-dl/issues/9965)
             'url': 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=AQ%7E%7E,AAABXlLMdok%7E,NJ4EoMlZ4rZdx9eU1rkMVd8EaYPBBUlg',
             'info_dict': {
                 'id': '1522758701001',
@@ -155,10 +155,10 @@ class BrightcoveLegacyIE(InfoExtractor):
         <object class="BrightcoveExperience">{params}</object>
         """
 
-        # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
+        # Fix up some stupid HTML, see https://github.com/ytdl-org/youtube-dl/issues/1553
         object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                             lambda m: m.group(1) + '/>', object_str)
-        # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
+        # Fix up some stupid XML, see https://github.com/ytdl-org/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')
         # remove namespace to simplify extraction
         object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 46380430f..1ec58f7d8 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -155,7 +155,7 @@ class CeskaTelevizeIE(InfoExtractor):
                         stream_formats = self._extract_mpd_formats(
                             stream_url, playlist_id,
                             mpd_id='dash-%s' % format_id, fatal=False)
-                    # See https://github.com/rg3/youtube-dl/issues/12119#issuecomment-280037031
+                    # See https://github.com/ytdl-org/youtube-dl/issues/12119#issuecomment-280037031
                     if format_id == 'audioDescription':
                         for f in stream_formats:
                             f['source_preference'] = -10
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4839edbf7..dfd0584d3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1454,7 +1454,7 @@ class InfoExtractor(object):
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
-            # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
+            # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244)
             transform_source=transform_source,
             fatal=fatal)
 
@@ -1485,7 +1485,7 @@ class InfoExtractor(object):
             manifest_version = '2.0'
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         # Remove unsupported DRM protected media from final formats
-        # rendition (see https://github.com/rg3/youtube-dl/issues/8573).
+        # rendition (see https://github.com/ytdl-org/youtube-dl/issues/8573).
         media_nodes = remove_encrypted_media(media_nodes)
         if not media_nodes:
             return formats
@@ -1615,8 +1615,8 @@ class InfoExtractor(object):
 
         # References:
         # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
-        # 2. https://github.com/rg3/youtube-dl/issues/12211
-        # 3. https://github.com/rg3/youtube-dl/issues/18923
+        # 2. https://github.com/ytdl-org/youtube-dl/issues/12211
+        # 3. https://github.com/ytdl-org/youtube-dl/issues/18923
 
         # We should try extracting formats only from master playlists [1, 4.3.4],
         # i.e. playlists that describe available qualities. On the other hand
@@ -2159,7 +2159,7 @@ class InfoExtractor(object):
                             # First of, % characters outside $...$ templates
                             # must be escaped by doubling for proper processing
                             # by % operator string formatting used further (see
-                            # https://github.com/rg3/youtube-dl/issues/16867).
+                            # https://github.com/ytdl-org/youtube-dl/issues/16867).
                             t = ''
                             in_template = False
                             for c in tmpl:
@@ -2178,7 +2178,7 @@ class InfoExtractor(object):
 
                         # @initialization is a regular template like @media one
                         # so it should be handled just the same way (see
-                        # https://github.com/rg3/youtube-dl/issues/11605)
+                        # https://github.com/ytdl-org/youtube-dl/issues/11605)
                         if 'initialization' in representation_ms_info:
                             initialization_template = prepare_template(
                                 'initialization',
@@ -2264,7 +2264,7 @@ class InfoExtractor(object):
                         elif 'segment_urls' in representation_ms_info:
                             # Segment URLs with no SegmentTimeline
                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
-                            # https://github.com/rg3/youtube-dl/pull/14844
+                            # https://github.com/ytdl-org/youtube-dl/pull/14844
                             fragments = []
                             segment_duration = float_or_none(
                                 representation_ms_info['segment_duration'],
@@ -2294,8 +2294,8 @@ class InfoExtractor(object):
                         # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
                         # is not necessarily unique within a Period thus formats with
                         # the same `format_id` are quite possible. There are numerous examples
-                        # of such manifests (see https://github.com/rg3/youtube-dl/issues/15111,
-                        # https://github.com/rg3/youtube-dl/issues/13919)
+                        # of such manifests (see https://github.com/ytdl-org/youtube-dl/issues/15111,
+                        # https://github.com/ytdl-org/youtube-dl/issues/13919)
                         full_info = formats_dict.get(representation_id, {}).copy()
                         full_info.update(f)
                         formats.append(full_info)
@@ -2456,7 +2456,7 @@ class InfoExtractor(object):
         media_tags.extend(re.findall(
             # We only allow video|audio followed by a whitespace or '>'.
             # Allowing more characters may end up in significant slow down (see
-            # https://github.com/rg3/youtube-dl/issues/11979, example URL:
+            # https://github.com/ytdl-org/youtube-dl/issues/11979, example URL:
             # http://www.porntrex.com/maps/videositemap.xml).
             r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
         for media_tag, media_type, media_content in media_tags:
diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index 79f7a9cd1..06d9148e0 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -36,7 +36,7 @@ class UnicodeBOMIE(InfoExtractor):
         _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
 
         # Disable test for python 3.2 since BOM is broken in re in this version
-        # (see https://github.com/rg3/youtube-dl/issues/9751)
+        # (see https://github.com/ytdl-org/youtube-dl/issues/9751)
         _TESTS = [] if (3, 0) < sys.version_info <= (3, 3) else [{
             'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
             'only_matching': True,
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 5948154f8..588c3c71b 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -107,7 +107,7 @@ class CrunchyrollBaseIE(InfoExtractor):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                    else sanitized_Request(url_or_request))
         # Accept-Language must be set explicitly to accept any language to avoid issues
-        # similar to https://github.com/rg3/youtube-dl/issues/6797.
+        # similar to https://github.com/ytdl-org/youtube-dl/issues/6797.
         # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
         # should be imposed or not (from what I can see it just takes the first language
         # ignoring the priority and requires it to correspond the IP). By the way this causes
@@ -124,7 +124,7 @@ class CrunchyrollBaseIE(InfoExtractor):
         # > This content may be inappropriate for some people.
         # > Are you sure you want to continue?
         # since it's not disabled by default in crunchyroll account's settings.
-        # See https://github.com/rg3/youtube-dl/issues/7202.
+        # See https://github.com/ytdl-org/youtube-dl/issues/7202.
         qs['skip_wall'] = ['1']
         return compat_urlparse.urlunparse(
             parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 1816c559e..1a2c1308a 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -166,11 +166,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'comment count', default=None))
 
         player_v5 = self._search_regex(
-            [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
+            [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/ytdl-org/youtube-dl/issues/7826
              r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
              r'buildPlayer\(({.+?})\);',
              r'var\s+config\s*=\s*({.+?});',
-             # New layout regex (see https://github.com/rg3/youtube-dl/issues/13580)
+             # New layout regex (see https://github.com/ytdl-org/youtube-dl/issues/13580)
              r'__PLAYER_CONFIG__\s*=\s*({.+?});'],
             webpage, 'player v5', default=None)
         if player_v5:
diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 8d31258c1..848d387d1 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -114,7 +114,7 @@ class DreiSatIE(InfoExtractor):
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
                 # the certificates are misconfigured (see
-                # https://github.com/rg3/youtube-dl/issues/8665)
+                # https://github.com/ytdl-org/youtube-dl/issues/8665)
                 if video_url.startswith('https://'):
                     continue
                 formats.extend(self._extract_m3u8_formats(
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 6101fb6bd..20f449c5c 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -143,7 +143,7 @@ class FranceTVIE(InfoExtractor):
             ext = determine_ext(video_url)
             if ext == 'f4m':
                 if georestricted:
-                    # See https://github.com/rg3/youtube-dl/issues/3963
+                    # See https://github.com/ytdl-org/youtube-dl/issues/3963
                     # m3u8 urls work fine
                     continue
                 formats.extend(self._extract_f4m_formats(
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 067de28cd..6f48b04da 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -430,7 +430,7 @@ class GenericIE(InfoExtractor):
             },
         },
         {
-            # https://github.com/rg3/youtube-dl/issues/2253
+            # https://github.com/ytdl-org/youtube-dl/issues/2253
             'url': 'http://bcove.me/i6nfkrc3',
             'md5': '0ba9446db037002366bab3b3eb30c88c',
             'info_dict': {
@@ -455,7 +455,7 @@ class GenericIE(InfoExtractor):
             },
         },
         {
-            # https://github.com/rg3/youtube-dl/issues/3541
+            # https://github.com/ytdl-org/youtube-dl/issues/3541
             'add_ie': ['BrightcoveLegacy'],
             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
             'info_dict': {
@@ -919,7 +919,7 @@ class GenericIE(InfoExtractor):
             }
         },
         # Multiple brightcove videos
-        # https://github.com/rg3/youtube-dl/issues/2283
+        # https://github.com/ytdl-org/youtube-dl/issues/2283
         {
             'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
             'info_dict': {
@@ -2371,7 +2371,7 @@ class GenericIE(InfoExtractor):
             return camtasia_res
 
         # Sometimes embedded video player is hidden behind percent encoding
-        # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
+        # (e.g. https://github.com/ytdl-org/youtube-dl/issues/2448)
         # Unescaping the whole page allows to handle those cases in a generic way
         webpage = compat_urllib_parse_unquote(webpage)
 
@@ -3177,7 +3177,7 @@ class GenericIE(InfoExtractor):
                     jwplayer_data, video_id, require_title=False, base_url=url)
                 return merge_dicts(info, info_dict)
             except ExtractorError:
-                # See https://github.com/rg3/youtube-dl/pull/16735
+                # See https://github.com/ytdl-org/youtube-dl/pull/16735
                 pass
 
         # Video.js embed
diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 3bf462d63..589e4d5c3 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -36,7 +36,7 @@ class GoogleDriveIE(InfoExtractor):
         }
     }, {
         # video can't be watched anonymously due to view count limit reached,
-        # but can be downloaded (see https://github.com/rg3/youtube-dl/issues/14046)
+        # but can be downloaded (see https://github.com/ytdl-org/youtube-dl/issues/14046)
         'url': 'https://drive.google.com/file/d/0B-vUyvmDLdWDcEt4WjBqcmI2XzQ/view',
         'md5': 'bfbd670d03a470bb1e6d4a257adec12e',
         'info_dict': {
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 63e10125e..cc5b2a1c1 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -215,7 +215,7 @@ class KuwoSingerIE(InfoExtractor):
             'title': 'Ali',
         },
         'playlist_mincount': 95,
-        'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
+        'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/ytdl-org/youtube-dl/jobs/78878540
     }]
 
     PAGE_SIZE = 15
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 22a067e40..5df14bb41 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -44,7 +44,7 @@ class LiveLeakIE(InfoExtractor):
         },
         'skip': 'Video is dead',
     }, {
-        # Covers https://github.com/rg3/youtube-dl/pull/5983
+        # Covers https://github.com/ytdl-org/youtube-dl/pull/5983
         # Multiple resolutions
         'url': 'http://www.liveleak.com/view?i=801_1409392012',
         'md5': 'c3a449dbaca5c0d1825caecd52a57d7b',
@@ -57,7 +57,7 @@ class LiveLeakIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$'
         }
     }, {
-        # Covers https://github.com/rg3/youtube-dl/pull/10664#issuecomment-247439521
+        # Covers https://github.com/ytdl-org/youtube-dl/pull/10664#issuecomment-247439521
         'url': 'http://m.liveleak.com/view?i=763_1473349649',
         'add_ie': ['Youtube'],
         'info_dict': {
@@ -130,7 +130,7 @@ class LiveLeakIE(InfoExtractor):
 
                 # Removing '.*.mp4' gives the raw video, which is essentially
                 # the same video without the LiveLeak logo at the top (see
-                # https://github.com/rg3/youtube-dl/pull/4768)
+                # https://github.com/ytdl-org/youtube-dl/pull/4768)
                 orig_url = re.sub(r'\.mp4\.[^.]+', '', a_format['url'])
                 if a_format['url'] != orig_url:
                     format_id = a_format.get('format_id')
diff --git a/youtube_dl/extractor/msn.py b/youtube_dl/extractor/msn.py
index 650731fdc..0460cf4d5 100644
--- a/youtube_dl/extractor/msn.py
+++ b/youtube_dl/extractor/msn.py
@@ -70,7 +70,7 @@ class MSNIE(InfoExtractor):
                 continue
             if 'm3u8' in format_url:
                 # m3u8_native should not be used here until
-                # https://github.com/rg3/youtube-dl/issues/9913 is fixed
+                # https://github.com/ytdl-org/youtube-dl/issues/9913 is fixed
                 m3u8_formats = self._extract_m3u8_formats(
                     format_url, display_id, 'mp4',
                     m3u8_id='hls', fatal=False)
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index cf440f713..eddfe1f37 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -108,7 +108,7 @@ class NHLIE(NHLBaseIE):
             'timestamp': 1454544904,
         },
     }, {
-        # Some m3u8 URLs are invalid (https://github.com/rg3/youtube-dl/issues/10713)
+        # Some m3u8 URLs are invalid (https://github.com/ytdl-org/youtube-dl/issues/10713)
         'url': 'https://www.nhl.com/predators/video/poile-laviolette-on-subban-trade/t-277437416/c-44315003',
         'md5': '50b2bb47f405121484dda3ccbea25459',
         'info_dict': {
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 58b371ed7..30df905af 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -115,7 +115,7 @@ class NocoIE(InfoExtractor):
 
         # Timestamp adjustment offset between server time and local time
         # must be calculated in order to use timestamps closest to server's
-        # in all API requests (see https://github.com/rg3/youtube-dl/issues/7864)
+        # in all API requests (see https://github.com/ytdl-org/youtube-dl/issues/7864)
         webpage = self._download_webpage(url, video_id)
 
         player_url = self._search_regex(
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 8ae5fadd8..3e44b7829 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -21,7 +21,7 @@ class OnceIE(InfoExtractor):
         progressive_formats = []
         for adaptive_format in formats:
             # Prevent advertisement from embedding into m3u8 playlist (see
-            # https://github.com/rg3/youtube-dl/issues/8893#issuecomment-199912684)
+            # https://github.com/ytdl-org/youtube-dl/issues/8893#issuecomment-199912684)
             adaptive_format['url'] = re.sub(
                 r'\badsegmentlength=\d+', r'adsegmentlength=0', adaptive_format['url'])
             rendition_id = self._search_regex(
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 80340f595..4dbe661be 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -305,7 +305,7 @@ class PBSIE(InfoExtractor):
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
             # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
-            # https://github.com/rg3/youtube-dl/issues/7059)
+            # https://github.com/ytdl-org/youtube-dl/issues/7059)
             'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
             'md5': '59b0ef5009f9ac8a319cc5efebcd865e',
             'info_dict': {
@@ -348,7 +348,7 @@ class PBSIE(InfoExtractor):
             },
         },
         {
-            # https://github.com/rg3/youtube-dl/issues/13801
+            # https://github.com/ytdl-org/youtube-dl/issues/13801
             'url': 'https://www.pbs.org/video/pbs-newshour-full-episode-july-31-2017-1501539057/',
             'info_dict': {
                 'id': '3003333873',
@@ -642,7 +642,7 @@ class PBSIE(InfoExtractor):
                 # we won't try extracting them.
                 # Since summer 2016 higher quality formats (4500k and 6500k) are also available
                 # albeit they are not documented in [2].
-                # 1. https://github.com/rg3/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
+                # 1. https://github.com/ytdl-org/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
                 # 2. https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
                 if not bitrate or int(bitrate) < 400:
                     continue
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index eafe56897..abd08bc28 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -327,7 +327,7 @@ query viewClip {
         )
 
         # Some courses also offer widescreen resolution for high quality (see
-        # https://github.com/rg3/youtube-dl/issues/7766)
+        # https://github.com/ytdl-org/youtube-dl/issues/7766)
         widescreen = course.get('supportsWideScreenVideoFormats') is True
         best_quality = 'high-widescreen' if widescreen else 'high'
         if widescreen:
@@ -388,8 +388,8 @@ query viewClip {
 
                 # Pluralsight tracks multiple sequential calls to ViewClip API and start
                 # to return 429 HTTP errors after some time (see
-                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
-                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
+                # https://github.com/ytdl-org/youtube-dl/pull/6989). Moreover it may even lead
+                # to account ban (see https://github.com/ytdl-org/youtube-dl/issues/6842).
                 # To somewhat reduce the probability of these consequences
                 # we will sleep random amount of time before each call to ViewClip.
                 self._sleep(
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 641083da7..3a474c179 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -332,7 +332,7 @@ class PornHubPlaylistBaseIE(PornHubBaseIE):
     def _extract_entries(self, webpage, host):
         # Only process container div with main playlist content skipping
         # drop-down menu that uses similar pattern for videos (see
-        # https://github.com/rg3/youtube-dl/issues/11594).
+        # https://github.com/ytdl-org/youtube-dl/issues/11594).
         container = self._search_regex(
             r'(?s)(<div[^>]+class=["\']container.+)', webpage,
             'container', default=webpage)
diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index d0955d079..7d11c2b9b 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -140,8 +140,8 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
 
     _TESTS = [
         {
-            # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242
-            # in response to fixing https://github.com/rg3/youtube-dl/issues/6215:
+            # Tests changes introduced in https://github.com/ytdl-org/youtube-dl/pull/6242
+            # in response to fixing https://github.com/ytdl-org/youtube-dl/issues/6215:
             # - malformed f4m manifest support
             # - proper handling of URLs starting with `https?://` in 2.0 manifests
             # - recursive child f4m manifests extraction
diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index dd95f99f2..2e3f1f58f 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -96,7 +96,7 @@ class RadioCanadaIE(InfoExtractor):
                         return text
 
         # protectionType does not necessarily mean the video is DRM protected (see
-        # https://github.com/rg3/youtube-dl/pull/18609).
+        # https://github.com/ytdl-org/youtube-dl/pull/18609).
         if get_meta('protectionType'):
             self.report_warning('This video is probably DRM protected.')
 
diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index be36acc46..0b5e55d16 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -45,7 +45,7 @@ class RtlNlIE(InfoExtractor):
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
     }, {
-        # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
+        # empty synopsis and missing episodes (see https://github.com/ytdl-org/youtube-dl/issues/6275)
         # best format available nettv
         'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
         'info_dict': {
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 90b351cbb..07055513a 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -209,7 +209,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             return [m.group('url')]
 
         # Are whitesapces ignored in URLs?
-        # https://github.com/rg3/youtube-dl/issues/12044
+        # https://github.com/ytdl-org/youtube-dl/issues/12044
         matches = re.findall(
             r'(?s)<(?:iframe|script)[^>]+src=(["\'])((?:https?:)?//player\.theplatform\.com/p/.+?)\1', webpage)
         if matches:
@@ -271,7 +271,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
 
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
-        # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
+        # Explicitly specified SMIL (see https://github.com/ytdl-org/youtube-dl/issues/7385)
         elif '/guid/' in url:
             headers = {}
             source_url = smuggled_data.get('source_url')
diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 25e1fd46d..8cc57b919 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -68,7 +68,7 @@ class TouTvIE(RadioCanadaIE):
         path = self._match_id(url)
         metadata = self._download_json('http://ici.tou.tv/presentation/%s' % path, path)
         # IsDrm does not necessarily mean the video is DRM protected (see
-        # https://github.com/rg3/youtube-dl/issues/13994).
+        # https://github.com/ytdl-org/youtube-dl/issues/13994).
         if metadata.get('IsDrm'):
             self.report_warning('This video is probably DRM protected.', path)
         video_id = metadata['IdMedia']
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index ae8de9897..66ea6fb15 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -375,7 +375,7 @@ class UdemyIE(InfoExtractor):
                     }, res))
 
             # react rendition since 2017.04.15 (see
-            # https://github.com/rg3/youtube-dl/issues/12744)
+            # https://github.com/ytdl-org/youtube-dl/issues/12744)
             data = self._parse_json(
                 self._search_regex(
                     r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 5737d4d16..582090d0d 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -165,7 +165,7 @@ class UstreamIE(InfoExtractor):
         m = re.match(self._VALID_URL, url)
         video_id = m.group('id')
 
-        # some sites use this embed format (see: https://github.com/rg3/youtube-dl/issues/2990)
+        # some sites use this embed format (see: https://github.com/ytdl-org/youtube-dl/issues/2990)
         if m.group('type') == 'embed/recorded':
             video_id = m.group('id')
             desktop_url = 'http://www.ustream.tv/recorded/' + video_id
diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 0d8d832cc..a6dc3c8d8 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -54,7 +54,7 @@ class VeeHDIE(InfoExtractor):
         video_id = self._match_id(url)
 
         # VeeHD seems to send garbage on the first request.
-        # See https://github.com/rg3/youtube-dl/issues/2102
+        # See https://github.com/ytdl-org/youtube-dl/issues/2102
         self._download_webpage(url, video_id, 'Requesting webpage')
         webpage = self._download_webpage(url, video_id)
 
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 890a149ea..4aa72cbd1 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -200,7 +200,7 @@ class VevoIE(VevoBaseIE):
             fatal=False)
 
         # Some videos are only available via webpage (e.g.
-        # https://github.com/rg3/youtube-dl/issues/9366)
+        # https://github.com/ytdl-org/youtube-dl/issues/9366)
         if not video_versions:
             webpage = self._download_webpage(url, video_id)
             json_data = self._extract_json(webpage, video_id)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index e3ec550f0..9aec8a2ab 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -530,7 +530,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         orig_url = url
         if mobj.group('pro'):
             # some videos require portfolio_id to be present in player url
-            # https://github.com/rg3/youtube-dl/issues/20070
+            # https://github.com/ytdl-org/youtube-dl/issues/20070
             url = self._extract_url(url, self._download_webpage(url, video_id))
         elif mobj.group('player'):
             url = 'https://player.vimeo.com/video/' + video_id
@@ -585,7 +585,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 if not config_url:
                     # Sometimes new react-based page is served instead of old one that require
                     # different config URL extraction approach (see
-                    # https://github.com/rg3/youtube-dl/pull/7209)
+                    # https://github.com/ytdl-org/youtube-dl/pull/7209)
                     vimeo_clip_page_config = self._search_regex(
                         r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
                         'vimeo clip page config')
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index b52d15ac6..d1fe95654 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -50,7 +50,7 @@ class VKBaseIE(InfoExtractor):
 
         # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header
         # and expects the first one to be set rather than second (see
-        # https://github.com/rg3/youtube-dl/issues/9841#issuecomment-227871201).
+        # https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201).
         # As of RFC6265 the newer one cookie should be set into cookie store
         # what actually happens.
         # We will workaround this VK issue by resetting the remixlhk cookie to
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 0b5165fd0..d68f222e7 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -239,7 +239,7 @@ class VLiveChannelIE(InfoExtractor):
                     # Large values of maxNumOfRows (~300 or above) may cause
                     # empty responses (see [1]), e.g. this happens for [2] that
                     # has more than 300 videos.
-                    # 1. https://github.com/rg3/youtube-dl/issues/13830
+                    # 1. https://github.com/ytdl-org/youtube-dl/issues/13830
                     # 2. http://channels.vlive.tv/EDBF.
                     'maxNumOfRows': 100,
                     '_': int(time.time()),
diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 009203851..456f95f69 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -76,7 +76,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
             track_id, 'Downloading track location JSON')
 
         # Each string is now wrapped in a list, this is probably only temporarily thus
-        # supporting both scenarios (see https://github.com/rg3/youtube-dl/issues/10193)
+        # supporting both scenarios (see https://github.com/ytdl-org/youtube-dl/issues/10193)
         for k, v in data.items():
             if v and isinstance(v, list):
                 data[k] = v[0]
@@ -200,7 +200,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
         'skip': 'Travis CI servers blocked by YandexMusic',
     }, {
         # playlist exceeding the limit of 150 tracks shipped with webpage (see
-        # https://github.com/rg3/youtube-dl/issues/6666)
+        # https://github.com/ytdl-org/youtube-dl/issues/6666)
         'url': 'https://music.yandex.ru/users/ya.playlist/playlists/1036',
         'info_dict': {
             'id': '1036',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 457e2acea..0bc6cc7a7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -430,7 +430,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
@@ -695,7 +695,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'age_limit': 18,
             },
         },
-        # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
+        # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
         # YouTube Red ad is not captured for creator
         {
             'url': '__2ABJjxzNo',
@@ -716,7 +716,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'DASH manifest missing',
             ]
         },
-        # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
+        # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
         {
             'url': 'lqQg6PlCWgI',
             'info_dict': {
@@ -767,7 +767,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'skip': 'This live event has ended.',
         },
-        # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
+        # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
         {
             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
             'info_dict': {
@@ -870,7 +870,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'skip': 'This video is not available.',
         },
         {
-            # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
+            # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
             'info_dict': {
                 'id': 'gVfLd0zydlo',
@@ -888,10 +888,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'only_matching': True,
         },
         {
-            # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
+            # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
             # Also tests cut-off URL expansion in video description (see
-            # https://github.com/rg3/youtube-dl/issues/1892,
-            # https://github.com/rg3/youtube-dl/issues/8164)
+            # https://github.com/ytdl-org/youtube-dl/issues/1892,
+            # https://github.com/ytdl-org/youtube-dl/issues/8164)
             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
             'info_dict': {
                 'id': 'lsguqyKfVQg',
@@ -913,7 +913,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
         },
         {
-            # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
+            # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
             'only_matching': True,
         },
@@ -977,7 +977,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'only_matching': True,
         },
         {
-            # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
+            # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
             'only_matching': True,
         },
@@ -1283,8 +1283,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # regex won't capture the whole JSON. Yet working around by trying more
             # concrete regex first keeping in mind proper quoted string handling
             # to be implemented in future that will replace this workaround (see
-            # https://github.com/rg3/youtube-dl/issues/7468,
-            # https://github.com/rg3/youtube-dl/pull/7599)
+            # https://github.com/ytdl-org/youtube-dl/issues/7468,
+            # https://github.com/ytdl-org/youtube-dl/pull/7599)
             r';ytplayer\.config\s*=\s*({.+?});ytplayer',
             r';ytplayer\.config\s*=\s*({.+?});',
         )
@@ -1599,7 +1599,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     add_dash_mpd(video_info)
                 # Rental video is not rented but preview is available (e.g.
                 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
-                # https://github.com/rg3/youtube-dl/issues/10532)
+                # https://github.com/ytdl-org/youtube-dl/issues/10532)
                 if not video_info and args.get('ypc_vid'):
                     return self.url_result(
                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
@@ -1619,7 +1619,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
                 # manifest pointed by get_video_info's dashmpd).
                 # The general idea is to take a union of itags of both DASH manifests (for example
-                # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+                # video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
                 self.report_video_info_webpage_download(video_id)
                 for el in ('info', 'embedded', 'detailpage', 'vevo', ''):
                     query = {
@@ -1654,7 +1654,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if 'token' in get_video_info:
                         # Different get_video_info requests may report different results, e.g.
                         # some may report video unavailability, but some may serve it without
-                        # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
+                        # any complaint (see https://github.com/ytdl-org/youtube-dl/issues/7362,
                         # the original webpage as well as el=info and el=embedded get_video_info
                         # requests report video unavailability due to geo restriction while
                         # el=detailpage succeeds and returns valid data). This is probably
@@ -1750,7 +1750,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     for feed in multifeed_metadata_list.split(','):
                         # Unquote should take place before split on comma (,) since textual
                         # fields may contain comma as well (see
-                        # https://github.com/rg3/youtube-dl/issues/8536)
+                        # https://github.com/ytdl-org/youtube-dl/issues/8536)
                         feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
                         entries.append({
                             '_type': 'url_transparent',
@@ -1775,7 +1775,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
-            raise ExtractorError('"rental" videos not supported. See https://github.com/rg3/youtube-dl/issues/359 for more information.', expected=True)
+            raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True)
 
         def _extract_filesize(media_url):
             return int_or_none(self._search_regex(
@@ -1792,7 +1792,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         elif not is_live and (len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
-                raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
+                raise ExtractorError('rtmpe downloads are not supported, see https://github.com/ytdl-org/youtube-dl/issues/343 for more information.', expected=True)
             formats_spec = {}
             fmt_list = video_info.get('fmt_list', [''])[0]
             if fmt_list:
@@ -1904,7 +1904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     dct.update(formats_spec[format_id])
 
                 # Some itags are not included in DASH manifest thus corresponding formats will
-                # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
+                # lack metadata (see https://github.com/ytdl-org/youtube-dl/pull/5993).
                 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
                 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
                 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
@@ -2155,7 +2155,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Remove the formats we found through non-DASH, they
                     # contain less info and it can be wrong, because we use
                     # fixed values (for example the resolution). See
-                    # https://github.com/rg3/youtube-dl/issues/5774 for an
+                    # https://github.com/ytdl-org/youtube-dl/issues/5774 for an
                     # example.
                     formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
                     formats.extend(dash_formats.values())
@@ -2424,7 +2424,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
 
-        # the yt-alert-message now has tabindex attribute (see https://github.com/rg3/youtube-dl/issues/11604)
+        # the yt-alert-message now has tabindex attribute (see https://github.com/ytdl-org/youtube-dl/issues/11604)
         for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
             match = match.strip()
             # Check if the playlist exists or is private
@@ -2517,7 +2517,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             return playlist
 
         # Some playlist URLs don't actually serve a playlist (see
-        # https://github.com/rg3/youtube-dl/issues/10537).
+        # https://github.com/ytdl-org/youtube-dl/issues/10537).
         # Fallback to plain video extraction if there is a video id
         # along with playlist id.
         return self.url_result(video_id, 'Youtube', video_id=video_id)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e7d8e8910..1ffabc62b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -45,7 +45,7 @@ def parseOpts(overrideArguments=None):
         except IOError:
             return default  # silently skip if file is not present
         try:
-            # FIXME: https://github.com/rg3/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
+            # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
             contents = optionf.read()
             if sys.version_info < (3,):
                 contents = contents.decode(preferredencoding())
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index ebce9666a..a14265c1b 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -31,7 +31,7 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose, opener):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = 'https://rg3.github.io/youtube-dl/update/'
+    UPDATE_URL = 'https://ytdl-org.github.io/youtube-dl/update/'
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a71eda85d..630dab8eb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -546,7 +546,7 @@ def sanitize_url(url):
         return 'http:%s' % url
     # Fix some common typos seen so far
     COMMON_TYPOS = (
-        # https://github.com/rg3/youtube-dl/issues/15649
+        # https://github.com/ytdl-org/youtube-dl/issues/15649
         (r'^httpss://', r'https://'),
         # https://bx1.be/lives/direct-tv/
         (r'^rmtp([es]?)://', r'rtmp\1://'),
@@ -596,7 +596,7 @@ def _htmlentity_transform(entity_with_semicolon):
             numstr = '0%s' % numstr
         else:
             base = 10
-        # See https://github.com/rg3/youtube-dl/issues/7518
+        # See https://github.com/ytdl-org/youtube-dl/issues/7518
         try:
             return compat_chr(int(numstr, base))
         except ValueError:
@@ -877,7 +877,7 @@ class XAttrUnavailableError(YoutubeDLError):
 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
     # expected HTTP responses to meet HTTP/1.0 or later (see also
-    # https://github.com/rg3/youtube-dl/issues/6727)
+    # https://github.com/ytdl-org/youtube-dl/issues/6727)
     if sys.version_info < (3, 0):
         kwargs['strict'] = True
     hc = http_class(*args, **compat_kwargs(kwargs))
@@ -1051,7 +1051,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
-        # https://github.com/rg3/youtube-dl/issues/6457).
+        # https://github.com/ytdl-org/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:
             location = resp.headers.get('Location')
             if location:
@@ -1190,7 +1190,7 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
     def http_response(self, request, response):
         # Python 2 will choke on next HTTP request in row if there are non-ASCII
         # characters in Set-Cookie HTTP header of last response (see
-        # https://github.com/rg3/youtube-dl/issues/6769).
+        # https://github.com/ytdl-org/youtube-dl/issues/6769).
         # In order to at least prevent crashing we will percent encode Set-Cookie
         # header before HTTPCookieProcessor starts processing it.
         # if sys.version_info < (3, 0) and response.headers:
@@ -2044,7 +2044,7 @@ def get_exe_version(exe, args=['--version'],
     try:
         # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
         # SIGTTOU if youtube-dl is run in the background.
-        # See https://github.com/rg3/youtube-dl/issues/955#issuecomment-209789656
+        # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
         out, _ = subprocess.Popen(
             [encodeArgument(exe)] + args,
             stdin=subprocess.PIPE,
@@ -2654,7 +2654,7 @@ def _match_one(filter_part, dct):
             # If the original field is a string and matching comparisonvalue is
             # a number we should respect the origin of the original field
             # and process comparison value as a string (see
-            # https://github.com/rg3/youtube-dl/issues/11082).
+            # https://github.com/ytdl-org/youtube-dl/issues/11082).
             actual_value is not None and m.group('intval') is not None and
                 isinstance(actual_value, compat_str)):
             if m.group('op') not in ('=', '!='):
@@ -3804,7 +3804,7 @@ def urshift(val, n):
 
 
 # Based on png2str() written by @gdkchan and improved by @yokrysty
-# Originally posted at https://github.com/rg3/youtube-dl/issues/9706
+# Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
 def decode_png(png_data):
     # Reference: https://www.w3.org/TR/PNG/
     header = png_data[8:]
@@ -3919,7 +3919,7 @@ def write_xattr(path, key, value):
         if hasattr(xattr, 'set'):  # pyxattr
             # Unicode arguments are not supported in python-pyxattr until
             # version 0.5.0
-            # See https://github.com/rg3/youtube-dl/issues/5498
+            # See https://github.com/ytdl-org/youtube-dl/issues/5498
             pyxattr_required_version = '0.5.0'
             if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
                 # TODO: fallback to CLI tools

From cdc7baffca7759397f4842934aa09311f58e80b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Mar 2019 04:33:27 +0700
Subject: [PATCH 472/510] [beeg] Add support for beeg.porn (closes #20306)

---
 youtube_dl/extractor/beeg.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 1086d7632..192f11ea6 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -9,8 +9,8 @@ from ..utils import (
 
 
 class BeegIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?beeg\.(?:com|porn(?:/video)?)/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://beeg.com/5416503',
         'md5': 'a1a1b1a8bc70a89e49ccfd113aed0820',
         'info_dict': {
@@ -24,7 +24,13 @@ class BeegIE(InfoExtractor):
             'tags': list,
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'https://beeg.porn/video/5416503',
+        'only_matching': True,
+    }, {
+        'url': 'https://beeg.porn/5416503',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 99534feaff3b437ddbf640cb2a69ae3900631a48 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 07:27:57 +0100
Subject: [PATCH 473/510] [videomega] remove extractor(closes #10108)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/videomega.py  | 60 ------------------------------
 2 files changed, 61 deletions(-)
 delete mode 100644 youtube_dl/extractor/videomega.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 923dfe7f4..43ff66ab8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1299,7 +1299,6 @@ from .viddler import ViddlerIE
 from .videa import VideaIE
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
-from .videomega import VideoMegaIE
 from .videomore import (
     VideomoreIE,
     VideomoreVideoIE,
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
deleted file mode 100644
index c02830ddd..000000000
--- a/youtube_dl/extractor/videomega.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    decode_packed_codes,
-    sanitized_Request,
-)
-
-
-class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
-    _TESTS = [{
-        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
-        'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
-        'info_dict': {
-            'id': 'AOSQBJYKIDDIKYJBQSOA',
-            'ext': 'mp4',
-            'title': '1254207',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }, {
-        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
-        'only_matching': True,
-    }, {
-        'url': 'http://videomega.tv/view.php?ref=090051111052065112106089103052052103089106112065052111051090',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
-        req = sanitized_Request(iframe_url)
-        req.add_header('Referer', url)
-        req.add_header('Cookie', 'noadvtday=0')
-        webpage = self._download_webpage(req, video_id)
-
-        title = self._html_search_regex(
-            r'<title>(.+?)</title>', webpage, 'title')
-        title = re.sub(
-            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
-        thumbnail = self._search_regex(
-            r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
-
-        real_codes = decode_packed_codes(webpage)
-        video_url = self._search_regex(
-            r'"src"\s*,\s*"([^"]+)"', real_codes, 'video URL')
-
-        return {
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-            'thumbnail': thumbnail,
-            'http_headers': {
-                'Referer': iframe_url,
-            },
-        }

From fddb4a3a8bdce63c03566eda3a96916d4cd02315 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 08:18:57 +0100
Subject: [PATCH 474/510] [escapist] Modernize

---
 youtube_dl/extractor/escapist.py | 35 ++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 4d8a3c134..4cd815ebc 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -1,14 +1,11 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     clean_html,
     int_or_none,
     float_or_none,
-    sanitized_Request,
 )
 
 
@@ -36,7 +33,7 @@ def _decrypt_config(key, string):
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'https?://?(?:www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
+    _VALID_URL = r'https?://?(?:(?:www|v1)\.)?escapistmagazine\.com/videos/view/[^/]+/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
@@ -61,6 +58,12 @@ class EscapistIE(InfoExtractor):
             'duration': 304,
             'uploader': 'The Escapist',
         }
+    }, {
+        'url': 'http://escapistmagazine.com/videos/view/the-escapist-presents/6618',
+        'only_matching': True,
+    }, {
+        'url': 'https://v1.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -74,19 +77,20 @@ class EscapistIE(InfoExtractor):
         video_id = ims_video['videoID']
         key = ims_video['hash']
 
-        config_req = sanitized_Request(
-            'http://www.escapistmagazine.com/videos/'
-            'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
-        config_req.add_header('Referer', url)
-        config = self._download_webpage(config_req, video_id, 'Downloading video config')
+        config = self._download_webpage(
+            'http://www.escapistmagazine.com/videos/vidconfig.php',
+            video_id, 'Downloading video config', headers={
+                'Referer': url,
+            }, query={
+                'videoID': video_id,
+                'hash': key,
+            })
 
-        data = json.loads(_decrypt_config(key, config))
+        data = self._parse_json(_decrypt_config(key, config), video_id)
 
         video_data = data['videoData']
 
         title = clean_html(video_data['title'])
-        duration = float_or_none(video_data.get('duration'), 1000)
-        uploader = video_data.get('publisher')
 
         formats = [{
             'url': video['src'],
@@ -99,8 +103,9 @@ class EscapistIE(InfoExtractor):
             'id': video_id,
             'formats': formats,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage) or data.get('poster'),
             'description': self._og_search_description(webpage),
-            'duration': duration,
-            'uploader': uploader,
+            'duration': float_or_none(video_data.get('duration'), 1000),
+            'uploader': video_data.get('publisher'),
+            'series': video_data.get('show'),
         }

From 197cf8b794a92727009bbbb8fd14bfca846f15c5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 09:04:06 +0100
Subject: [PATCH 475/510] [veoh] fix extraction

---
 youtube_dl/extractor/veoh.py | 99 +++++++++++++-----------------------
 1 file changed, 36 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 071774a6f..1c44c145c 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -1,13 +1,10 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    ExtractorError,
-    sanitized_Request,
+    parse_duration,
+    qualities,
 )
 
 
@@ -16,9 +13,9 @@ class VeohIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
-        'md5': '620e68e6a3cff80086df3348426c9ca3',
+        'md5': '9e7ecc0fd8bbee7a69fe38953aeebd30',
         'info_dict': {
-            'id': '56314296',
+            'id': 'v56314296nk7Zdmz3',
             'ext': 'mp4',
             'title': 'Straight Backs Are Stronger',
             'uploader': 'LUMOback',
@@ -56,29 +53,6 @@ class VeohIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _extract_formats(self, source):
-        formats = []
-        link = source.get('aowPermalink')
-        if link:
-            formats.append({
-                'url': link,
-                'ext': 'mp4',
-                'format_id': 'aow',
-            })
-        link = source.get('fullPreviewHashLowPath')
-        if link:
-            formats.append({
-                'url': link,
-                'format_id': 'low',
-            })
-        link = source.get('fullPreviewHashHighPath')
-        if link:
-            formats.append({
-                'url': link,
-                'format_id': 'high',
-            })
-        return formats
-
     def _extract_video(self, source):
         return {
             'id': source.get('videoId'),
@@ -93,38 +67,37 @@ class VeohIE(InfoExtractor):
         }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        video = self._download_json(
+            'https://www.veoh.com/watch/getVideo/' + video_id,
+            video_id)['video']
+        title = video['title']
 
-        if video_id.startswith('v'):
-            rsp = self._download_xml(
-                r'http://www.veoh.com/api/findByPermalink?permalink=%s' % video_id, video_id, 'Downloading video XML')
-            stat = rsp.get('stat')
-            if stat == 'ok':
-                return self._extract_video(rsp.find('./videoList/video'))
-            elif stat == 'fail':
-                raise ExtractorError(
-                    '%s said: %s' % (self.IE_NAME, rsp.find('./errorList/error').get('errorMessage')), expected=True)
+        thumbnail_url = None
+        q = qualities(['HQ', 'Regular'])
+        formats = []
+        for f_id, f_url in video.get('src', {}).items():
+            if not f_url:
+                continue
+            if f_id == 'poster':
+                thumbnail_url = f_url
+            else:
+                formats.append({
+                    'format_id': f_id,
+                    'quality': q(f_id),
+                    'url': f_url,
+                })
+        self._sort_formats(formats)
 
-        webpage = self._download_webpage(url, video_id)
-        age_limit = 0
-        if 'class="adultwarning-container"' in webpage:
-            self.report_age_confirmation()
-            age_limit = 18
-            request = sanitized_Request(url)
-            request.add_header('Cookie', 'confirmedAdult=true')
-            webpage = self._download_webpage(request, video_id)
-
-        m_youtube = re.search(r'http://www\.youtube\.com/v/(.*?)(\&|"|\?)', webpage)
-        if m_youtube is not None:
-            youtube_id = m_youtube.group(1)
-            self.to_screen('%s: detected Youtube video.' % video_id)
-            return self.url_result(youtube_id, 'Youtube')
-
-        info = json.loads(
-            self._search_regex(r'videoDetailsJSON = \'({.*?})\';', webpage, 'info').replace('\\\'', '\''))
-
-        video = self._extract_video(info)
-        video['age_limit'] = age_limit
-
-        return video
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video.get('description'),
+            'thumbnail': thumbnail_url,
+            'uploader': video.get('author', {}).get('nickname'),
+            'duration': int_or_none(video.get('lengthBySec')) or parse_duration(video.get('length')),
+            'view_count': int_or_none(video.get('views')),
+            'formats': formats,
+            'average_rating': int_or_none(video.get('rating')),
+            'comment_count': int_or_none(video.get('numOfComments')),
+        }

From f517c0c0a72c0af38bf7136b1519d115557dab2f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 09:43:06 +0100
Subject: [PATCH 476/510] [hypem] Modernize and extract more metadata(closes
 #15320)

---
 youtube_dl/extractor/hypem.py | 46 +++++++++++++----------------------
 1 file changed, 17 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index f7c913054..9ca28d632 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -1,18 +1,11 @@
 from __future__ import unicode_literals
 
-import json
-import time
-
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-)
+from ..utils import int_or_none
 
 
 class HypemIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://(?:www\.)?hypem\.com/track/(?P<id>[0-9a-z]{5})'
     _TEST = {
         'url': 'http://hypem.com/track/1v6ga/BODYWORK+-+TAME',
         'md5': 'b9cc91b5af8995e9f0c1cee04c575828',
@@ -21,41 +14,36 @@ class HypemIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'Tame',
             'uploader': 'BODYWORK',
+            'timestamp': 1371810457,
+            'upload_date': '20130621',
         }
     }
 
     def _real_extract(self, url):
         track_id = self._match_id(url)
 
-        data = {'ax': 1, 'ts': time.time()}
-        request = sanitized_Request(url + '?' + compat_urllib_parse_urlencode(data))
-        response, urlh = self._download_webpage_handle(
-            request, track_id, 'Downloading webpage with the url')
+        response = self._download_webpage(url, track_id)
 
-        html_tracks = self._html_search_regex(
-            r'(?ms)<script type="application/json" id="displayList-data">(.+?)</script>',
-            response, 'tracks')
-        try:
-            track_list = json.loads(html_tracks)
-            track = track_list['tracks'][0]
-        except ValueError:
-            raise ExtractorError('Hypemachine contained invalid JSON.')
+        track = self._parse_json(self._html_search_regex(
+            r'(?s)<script\s+type="application/json"\s+id="displayList-data">(.+?)</script>',
+            response, 'tracks'), track_id)['tracks'][0]
 
-        key = track['key']
         track_id = track['id']
         title = track['song']
 
-        request = sanitized_Request(
-            'http://hypem.com/serve/source/%s/%s' % (track_id, key),
-            '', {'Content-Type': 'application/json'})
-        song_data = self._download_json(request, track_id, 'Downloading metadata')
-        final_url = song_data['url']
-        artist = track.get('artist')
+        final_url = self._download_json(
+            'http://hypem.com/serve/source/%s/%s' % (track_id, track['key']),
+            track_id, 'Downloading metadata', headers={
+                'Content-Type': 'application/json'
+            })['url']
 
         return {
             'id': track_id,
             'url': final_url,
             'ext': 'mp3',
             'title': title,
-            'uploader': artist,
+            'uploader': track.get('artist'),
+            'duration': int_or_none(track.get('time')),
+            'timestamp': int_or_none(track.get('ts')),
+            'track': title,
         }

From 612a159510edffbf50e01c482b9ed2c947ed0112 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 09:51:26 +0100
Subject: [PATCH 477/510] [primesharetv] remove extractor

---
 youtube_dl/extractor/extractors.py   |  1 -
 youtube_dl/extractor/primesharetv.py | 62 ----------------------------
 2 files changed, 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/primesharetv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 43ff66ab8..687994045 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -901,7 +901,6 @@ from .puhutv import (
     PuhuTVSerieIE,
 )
 from .presstv import PressTVIE
-from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
deleted file mode 100644
index 0c1024772..000000000
--- a/youtube_dl/extractor/primesharetv.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-    urlencode_postdata,
-)
-
-
-class PrimeShareTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>[\da-zA-Z]+)'
-
-    _TEST = {
-        'url': 'http://primeshare.tv/download/238790B611',
-        'md5': 'b92d9bf5461137c36228009f31533fbc',
-        'info_dict': {
-            'id': '238790B611',
-            'ext': 'mp4',
-            'title': 'Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        if '>File not exist<' in webpage:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        fields = self._hidden_inputs(webpage)
-
-        headers = {
-            'Referer': url,
-            'Content-Type': 'application/x-www-form-urlencoded',
-        }
-
-        wait_time = int(self._search_regex(
-            r'var\s+cWaitTime\s*=\s*(\d+)',
-            webpage, 'wait time', default=7)) + 1
-        self._sleep(wait_time, video_id)
-
-        req = sanitized_Request(
-            url, urlencode_postdata(fields), headers)
-        video_page = self._download_webpage(
-            req, video_id, 'Downloading video page')
-
-        video_url = self._search_regex(
-            r"url\s*:\s*'([^']+\.primeshare\.tv(?::443)?/file/[^']+)'",
-            video_page, 'video url')
-
-        title = self._html_search_regex(
-            r'<h1>Watch\s*(?:&nbsp;)?\s*\((.+?)(?:\s*\[\.\.\.\])?\)\s*(?:&nbsp;)?\s*<strong>',
-            video_page, 'title')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'ext': 'mp4',
-        }

From 8569058f88f2abbc8c7377439d36fc13532bdbe3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 15:18:09 +0100
Subject: [PATCH 478/510] [moevideo] fix extraction

---
 youtube_dl/extractor/moevideo.py | 65 ++++++++------------------------
 1 file changed, 15 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 44bcc4982..eb9b4ce7c 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -1,15 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    clean_html,
     int_or_none,
-    sanitized_Request,
-    urlencode_postdata,
 )
 
 
@@ -17,8 +14,8 @@ class MoeVideoIE(InfoExtractor):
     IE_DESC = 'LetitBit video services: moevideo.net, playreplay.net and videochart.net'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-        (?:(?:moevideo|playreplay|videochart)\.net))/
-        (?:video|framevideo)/(?P<id>[0-9]+\.[0-9A-Za-z]+)'''
+        (?:(?:moevideo|playreplay|videochart)\.net|thesame\.tv))/
+        (?:video|framevideo|embed)/(?P<id>[0-9a-z]+\.[0-9A-Za-z]+)'''
     _API_URL = 'http://api.letitbit.net/'
     _API_KEY = 'tVL0gjqo5'
     _TESTS = [
@@ -57,58 +54,26 @@ class MoeVideoIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        host, video_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(
-            'http://%s/video/%s' % (mobj.group('host'), video_id),
+            'http://%s/video/%s' % (host, video_id),
             video_id, 'Downloading webpage')
 
         title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
 
-        r = [
-            self._API_KEY,
-            [
-                'preview/flv_link',
-                {
-                    'uid': video_id,
-                },
-            ],
-        ]
-        r_json = json.dumps(r)
-        post = urlencode_postdata({'r': r_json})
-        req = sanitized_Request(self._API_URL, post)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        response = self._download_json(req, video_id)
-        if response['status'] != 'OK':
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, response['data']),
-                expected=True
-            )
-        item = response['data'][0]
-        video_url = item['link']
-        duration = int_or_none(item['length'])
-        width = int_or_none(item['width'])
-        height = int_or_none(item['height'])
-        filesize = int_or_none(item['convert_size'])
-
-        formats = [{
-            'format_id': 'sd',
-            'http_headers': {'Range': 'bytes=0-'},  # Required to download
-            'url': video_url,
-            'width': width,
-            'height': height,
-            'filesize': filesize,
-        }]
+        embed_webpage = self._download_webpage(
+            'http://%s/embed/%s' % (host, video_id),
+            video_id, 'Downloading embed webpage')
+        video = self._parse_json(self._search_regex(
+            r'mvplayer\("#player"\s*,\s*({.+})',
+            embed_webpage, 'mvplayer'), video_id)['video']
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
-            'description': description,
-            'duration': duration,
-            'formats': formats,
+            'thumbnail': video.get('poster') or self._og_search_thumbnail(webpage),
+            'description': clean_html(self._og_search_description(webpage)),
+            'duration': int_or_none(self._og_search_property('video:duration', webpage)),
+            'url': video['ourUrl'],
         }

From 4993132344611e77e71d27cbd4803a323e7006f4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 15:37:19 +0100
Subject: [PATCH 479/510] [pandatv] fix condition syntax

---
 youtube_dl/extractor/pandatv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index 13a2e7efc..4219802d5 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -36,7 +36,7 @@ class PandaTVIE(InfoExtractor):
             'https://www.panda.tv/api_room_v2?roomid=%s' % video_id, video_id)
 
         error_code = config.get('errno', 0)
-        if error_code is not 0:
+        if error_code != 0:
             raise ExtractorError(
                 '%s returned error %s: %s'
                 % (self.IE_NAME, error_code, config['errmsg']),

From e585547284649c19949db36b0982b7b245d39384 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Mar 2019 15:53:44 +0100
Subject: [PATCH 480/510] [viddler] Modernize

---
 youtube_dl/extractor/viddler.py | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 67808e7e6..642358433 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -1,19 +1,16 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlencode,
-    compat_urlparse,
-)
 from ..utils import (
     float_or_none,
     int_or_none,
-    sanitized_Request,
 )
 
 
 class ViddlerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)(?:.+?\bsecret=(\d+))?'
     _TESTS = [{
         'url': 'http://www.viddler.com/v/43903784',
         'md5': '9eee21161d2c7f5b39690c3e325fab2f',
@@ -78,23 +75,18 @@ class ViddlerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id, secret = re.match(self._VALID_URL, url).groups()
 
         query = {
             'video_id': video_id,
             'key': 'v0vhrt7bg2xq1vyxhkct',
         }
-
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        secret = qs.get('secret', [None])[0]
         if secret:
             query['secret'] = secret
 
-        headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
-        request = sanitized_Request(
-            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?%s'
-            % compat_urllib_parse_urlencode(query), None, headers)
-        data = self._download_json(request, video_id)['video']
+        data = self._download_json(
+            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json',
+            video_id, headers={'Referer': url}, query=query)['video']
 
         formats = []
         for filed in data['files']:

From e7e3ec828b3e0a400b7e9f26f066b8d4f09ce86e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Mar 2019 08:08:54 +0100
Subject: [PATCH 481/510] [biqle] fix extraction(closes #11471)(closes #15313)

---
 youtube_dl/extractor/biqle.py | 81 +++++++++++++++++++++++++++++------
 1 file changed, 69 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/biqle.py b/youtube_dl/extractor/biqle.py
index beaebfd2a..4c5c6be10 100644
--- a/youtube_dl/extractor/biqle.py
+++ b/youtube_dl/extractor/biqle.py
@@ -2,39 +2,96 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .vk import VKIE
+from ..utils import (
+    HEADRequest,
+    int_or_none,
+)
 
 
 class BIQLEIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?biqle\.(?:com|org|ru)/watch/(?P<id>-?\d+_\d+)'
     _TESTS = [{
-        'url': 'http://www.biqle.ru/watch/847655_160197695',
-        'md5': 'ad5f746a874ccded7b8f211aeea96637',
+        # Youtube embed
+        'url': 'https://biqle.ru/watch/-115995369_456239081',
+        'md5': '97af5a06ee4c29bbf9c001bdb1cf5c06',
         'info_dict': {
-            'id': '160197695',
+            'id': '8v4f-avW-VI',
             'ext': 'mp4',
-            'title': 'Foo Fighters - The Pretender (Live at Wembley Stadium)',
-            'uploader': 'Andrey Rogozin',
-            'upload_date': '20110605',
-        }
+            'title': "PASSE-PARTOUT - L'ete c'est fait pour jouer",
+            'description': 'Passe-Partout',
+            'uploader_id': 'mrsimpsonstef3',
+            'uploader': 'Phanolito',
+            'upload_date': '20120822',
+        },
     }, {
-        'url': 'https://biqle.org/watch/-44781847_168547604',
+        'url': 'http://biqle.org/watch/-44781847_168547604',
         'md5': '7f24e72af1db0edf7c1aaba513174f97',
         'info_dict': {
             'id': '168547604',
             'ext': 'mp4',
             'title': 'Ребенок в шоке от автоматической мойки',
+            'timestamp': 1396633454,
             'uploader': 'Dmitry Kotov',
+            'upload_date': '20140404',
+            'uploader_id': '47850140',
         },
-        'skip': ' This video was marked as adult.  Embedding adult videos on external sites is prohibited.',
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         embed_url = self._proto_relative_url(self._search_regex(
-            r'<iframe.+?src="((?:http:)?//daxab\.com/[^"]+)".*?></iframe>', webpage, 'embed url'))
+            r'<iframe.+?src="((?:https?:)?//daxab\.com/[^"]+)".*?></iframe>',
+            webpage, 'embed url'))
+        if VKIE.suitable(embed_url):
+            return self.url_result(embed_url, VKIE.ie_key(), video_id)
+
+        self._request_webpage(
+            HEADRequest(embed_url), video_id, headers={'Referer': url})
+        video_id, sig, _, access_token = self._get_cookies(embed_url)['video_ext'].value.split('%3A')
+        item = self._download_json(
+            'https://api.vk.com/method/video.get', video_id,
+            headers={'User-Agent': 'okhttp/3.4.1'}, query={
+                'access_token': access_token,
+                'sig': sig,
+                'v': 5.44,
+                'videos': video_id,
+            })['response']['items'][0]
+        title = item['title']
+
+        formats = []
+        for f_id, f_url in item.get('files', {}).items():
+            if f_id == 'external':
+                return self.url_result(f_url)
+            ext, height = f_id.split('_')
+            formats.append({
+                'format_id': height + 'p',
+                'url': f_url,
+                'height': int_or_none(height),
+                'ext': ext,
+            })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for k, v in item.items():
+            if k.startswith('photo_') and v:
+                width = k.replace('photo_', '')
+                thumbnails.append({
+                    'id': width,
+                    'url': v,
+                    'width': int_or_none(width),
+                })
 
         return {
-            '_type': 'url_transparent',
-            'url': embed_url,
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'comment_count': int_or_none(item.get('comments')),
+            'description': item.get('description'),
+            'duration': int_or_none(item.get('duration')),
+            'thumbnails': thumbnails,
+            'timestamp': int_or_none(item.get('date')),
+            'uploader': item.get('owner_id'),
+            'view_count': int_or_none(item.get('views')),
         }

From 68867668cf273c96d4a3d7429fb294f036f56437 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Mar 2019 16:52:28 +0100
Subject: [PATCH 482/510] [npr] fix extraction(closes #10793)(closes #13440)

---
 youtube_dl/extractor/npr.py | 76 +++++++++++++++++++++++++------------
 1 file changed, 51 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
index 1777aa10b..a5e8baa7e 100644
--- a/youtube_dl/extractor/npr.py
+++ b/youtube_dl/extractor/npr.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     int_or_none,
     qualities,
@@ -9,16 +8,16 @@ from ..utils import (
 
 
 class NprIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?npr\.org/(?:sections/[^/]+/)?\d{4}/\d{2}/\d{2}/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
+        'url': 'https://www.npr.org/sections/allsongs/2015/10/21/449974205/new-music-from-beach-house-chairlift-cmj-discoveries-and-more',
         'info_dict': {
             'id': '449974205',
             'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
         },
         'playlist_count': 7,
     }, {
-        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?action=1&t=1&islist=false&id=446928052&m=446929930&live=1',
+        'url': 'https://www.npr.org/sections/deceptivecadence/2015/10/09/446928052/music-from-the-shadows-ancient-armenian-hymns-and-piano-jazz',
         'info_dict': {
             'id': '446928052',
             'title': "Songs We Love: Tigran Hamasyan, 'Your Mercy is Boundless'"
@@ -32,30 +31,46 @@ class NprIE(InfoExtractor):
                 'duration': 402,
             },
         }],
+    }, {
+        # mutlimedia, not media title
+        'url': 'https://www.npr.org/2017/06/19/533198237/tigers-jaw-tiny-desk-concert',
+        'info_dict': {
+            'id': '533198237',
+            'title': 'Tigers Jaw: Tiny Desk Concert',
+        },
+        'playlist': [{
+            'md5': '12fa60cb2d3ed932f53609d4aeceabf1',
+            'info_dict': {
+                'id': '533201718',
+                'ext': 'mp4',
+                'title': 'Tigers Jaw: Tiny Desk Concert',
+                'duration': 402,
+            },
+        }],
+        'expected_warnings': ['Failed to download m3u8 information'],
     }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
-        config = self._download_json(
-            'http://api.npr.org/query?%s' % compat_urllib_parse_urlencode({
+        story = self._download_json(
+            'http://api.npr.org/query', playlist_id, query={
                 'id': playlist_id,
-                'fields': 'titles,audio,show',
+                'fields': 'audio,multimedia,title',
                 'format': 'json',
                 'apiKey': 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010',
-            }), playlist_id)
+            })['list']['story'][0]
+        playlist_title = story.get('title', {}).get('$text')
 
-        story = config['list']['story'][0]
-
-        KNOWN_FORMATS = ('threegp', 'mp4', 'mp3')
+        KNOWN_FORMATS = ('threegp', 'm3u8', 'smil', 'mp4', 'mp3')
         quality = qualities(KNOWN_FORMATS)
 
         entries = []
-        for audio in story.get('audio', []):
-            title = audio.get('title', {}).get('$text')
-            duration = int_or_none(audio.get('duration', {}).get('$text'))
+        for media in story.get('audio', []) + story.get('multimedia', []):
+            media_id = media['id']
+
             formats = []
-            for format_id, formats_entry in audio.get('format', {}).items():
+            for format_id, formats_entry in media.get('format', {}).items():
                 if not formats_entry:
                     continue
                 if isinstance(formats_entry, list):
@@ -64,19 +79,30 @@ class NprIE(InfoExtractor):
                 if not format_url:
                     continue
                 if format_id in KNOWN_FORMATS:
-                    formats.append({
-                        'url': format_url,
-                        'format_id': format_id,
-                        'ext': formats_entry.get('type'),
-                        'quality': quality(format_id),
-                    })
+                    if format_id == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            format_url, media_id, 'mp4', 'm3u8_native',
+                            m3u8_id='hls', fatal=False))
+                    elif format_id == 'smil':
+                        smil_formats = self._extract_smil_formats(
+                            format_url, media_id, transform_source=lambda s: s.replace(
+                                'rtmp://flash.npr.org/ondemand/', 'https://ondemand.npr.org/'))
+                        self._check_formats(smil_formats, media_id)
+                        formats.extend(smil_formats)
+                    else:
+                        formats.append({
+                            'url': format_url,
+                            'format_id': format_id,
+                            'quality': quality(format_id),
+                        })
             self._sort_formats(formats)
+
             entries.append({
-                'id': audio['id'],
-                'title': title,
-                'duration': duration,
+                'id': media_id,
+                'title': media.get('title', {}).get('$text') or playlist_title,
+                'thumbnail': media.get('altImageUrl', {}).get('$text'),
+                'duration': int_or_none(media.get('duration', {}).get('$text')),
                 'formats': formats,
             })
 
-        playlist_title = story.get('title', {}).get('$text')
         return self.playlist_result(entries, playlist_id, playlist_title)

From 47f9792620e06c3b9ac24d402951bce01ae5c038 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Mar 2019 22:55:32 +0700
Subject: [PATCH 483/510] [ruleporn] Remove extractor (closes #15344, closes
 #20324) Covered by generic extractor

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/ruleporn.py   | 44 ------------------------------
 2 files changed, 45 deletions(-)
 delete mode 100644 youtube_dl/extractor/ruleporn.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 687994045..884d25f50 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -977,7 +977,6 @@ from .rtvnh import RTVNHIE
 from .rtvs import RTVSIE
 from .rudo import RudoIE
 from .ruhd import RUHDIE
-from .ruleporn import RulePornIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/youtube_dl/extractor/ruleporn.py b/youtube_dl/extractor/ruleporn.py
deleted file mode 100644
index ebf9808d5..000000000
--- a/youtube_dl/extractor/ruleporn.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from __future__ import unicode_literals
-
-from .nuevo import NuevoBaseIE
-
-
-class RulePornIE(NuevoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[^/?#&]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'http://ruleporn.com/brunette-nympho-chick-takes-her-boyfriend-in-every-angle/',
-        'md5': '86861ebc624a1097c7c10eaf06d7d505',
-        'info_dict': {
-            'id': '48212',
-            'display_id': 'brunette-nympho-chick-takes-her-boyfriend-in-every-angle',
-            'ext': 'mp4',
-            'title': 'Brunette Nympho Chick Takes Her Boyfriend In Every Angle',
-            'description': 'md5:6d28be231b981fff1981deaaa03a04d5',
-            'age_limit': 18,
-            'duration': 635.1,
-        }
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._search_regex(
-            r'lovehomeporn\.com/embed/(\d+)', webpage, 'video id')
-
-        title = self._search_regex(
-            r'<h2[^>]+title=(["\'])(?P<url>.+?)\1',
-            webpage, 'title', group='url')
-        description = self._html_search_meta('description', webpage)
-
-        info = self._extract_nuevo(
-            'http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id,
-            video_id)
-        info.update({
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'age_limit': 18
-        })
-        return info

From 6db03a29d139aa7594b24117b912b589c7683a96 Mon Sep 17 00:00:00 2001
From: charon2019 <muenster2011@hotmail.com>
Date: Wed, 13 Mar 2019 10:47:55 +0100
Subject: [PATCH 484/510] [anitube] Remove extractor

site no longer exists
---
 youtube_dl/extractor/anitube.py    | 30 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 31 deletions(-)
 delete mode 100644 youtube_dl/extractor/anitube.py

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
deleted file mode 100644
index 2fd912da4..000000000
--- a/youtube_dl/extractor/anitube.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from __future__ import unicode_literals
-
-from .nuevo import NuevoBaseIE
-
-
-class AnitubeIE(NuevoBaseIE):
-    IE_NAME = 'anitube.se'
-    _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://www.anitube.se/video/36621',
-        'md5': '59d0eeae28ea0bc8c05e7af429998d43',
-        'info_dict': {
-            'id': '36621',
-            'ext': 'mp4',
-            'title': 'Recorder to Randoseru 01',
-            'duration': 180.19,
-        },
-        'skip': 'Blocked in the US',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        key = self._search_regex(
-            r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
-
-        return self._extract_nuevo(
-            'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, video_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 884d25f50..9a0459de8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -38,7 +38,6 @@ from .alphaporno import AlphaPornoIE
 from .amcnetworks import AMCNetworksIE
 from .americastestkitchen import AmericasTestKitchenIE
 from .animeondemand import AnimeOnDemandIE
-from .anitube import AnitubeIE
 from .anvato import AnvatoIE
 from .anysex import AnySexIE
 from .aol import AolIE

From 79d2077edc9c4ac8c71b672d8eff06ef847842c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Mar 2019 00:42:14 +0700
Subject: [PATCH 485/510] [extractor/common] Fix url meta field for
 unfragmented DASH formats (closes #20346)

---
 youtube_dl/extractor/common.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dfd0584d3..c291bc1df 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,10 +108,13 @@ class InfoExtractor(object):
                                    for RTMP - RTMP URL,
                                    for HLS - URL of the M3U8 media playlist,
                                    for HDS - URL of the F4M manifest,
-                                   for DASH - URL of the MPD manifest or
-                                              base URL representing the media
-                                              if MPD manifest is parsed from
-                                              a string,
+                                   for DASH
+                                     - HTTP URL to plain file media (in case of
+                                       unfragmented media)
+                                     - URL of the MPD manifest or base URL
+                                       representing the media if MPD manifest
+                                       is parsed froma string (in case of
+                                       fragmented media)
                                    for MSS - URL of the ISM manifest.
                     * manifest_url
                                  The URL of the manifest file in case of
@@ -2137,8 +2140,6 @@ class InfoExtractor(object):
                         bandwidth = int_or_none(representation_attrib.get('bandwidth'))
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
-                            # NB: mpd_url may be empty when MPD manifest is parsed from a string
-                            'url': mpd_url or base_url,
                             'manifest_url': mpd_url,
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
@@ -2277,10 +2278,14 @@ class InfoExtractor(object):
                                     fragment['duration'] = segment_duration
                                 fragments.append(fragment)
                             representation_ms_info['fragments'] = fragments
-                        # NB: MPD manifest may contain direct URLs to unfragmented media.
-                        # No fragments key is present in this case.
+                        # If there is a fragments key available then we correctly recognized fragmented media.
+                        # Otherwise we will assume unfragmented media with direct access. Technically, such
+                        # assumption is not necessarily correct since we may simply have no support for
+                        # some forms of fragmented media renditions yet, but for now we'll use this fallback.
                         if 'fragments' in representation_ms_info:
                             f.update({
+                                # NB: mpd_url may be empty when MPD manifest is parsed from a string
+                                'url': mpd_url or base_url,
                                 'fragment_base_url': base_url,
                                 'fragments': [],
                                 'protocol': 'http_dash_segments',
@@ -2291,6 +2296,10 @@ class InfoExtractor(object):
                                     f['url'] = initialization_url
                                 f['fragments'].append({location_key(initialization_url): initialization_url})
                             f['fragments'].extend(representation_ms_info['fragments'])
+                        else:
+                            # Assuming direct URL to unfragmented media.
+                            f['url'] = base_url
+
                         # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
                         # is not necessarily unique within a Period thus formats with
                         # the same `format_id` are quite possible. There are numerous examples

From 2e27421c70bcd71ea7e7d7fe8b7e468731334d51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Mar 2019 01:20:24 +0700
Subject: [PATCH 486/510] [test_InfoExtractor] Add test for #20346

---
 test/test_InfoExtractor.py         | 56 +++++++++++++++++++++++++++---
 test/testdata/mpd/unfragmented.mpd | 28 +++++++++++++++
 2 files changed, 80 insertions(+), 4 deletions(-)
 create mode 100644 test/testdata/mpd/unfragmented.mpd

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index da6cd39b6..d23d94349 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -574,7 +574,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 # Also tests duplicate representation ids, see
                 # https://github.com/ytdl-org/youtube-dl/issues/15111
                 'float_duration',
-                'http://unknown/manifest.mpd',
+                'http://unknown/manifest.mpd',  # mpd_url
+                None,  # mpd_base_url
                 [{
                     'manifest_url': 'http://unknown/manifest.mpd',
                     'ext': 'm4a',
@@ -654,7 +655,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
             ), (
                 # https://github.com/ytdl-org/youtube-dl/pull/14844
                 'urls_only',
-                'http://unknown/manifest.mpd',
+                'http://unknown/manifest.mpd',  # mpd_url
+                None,  # mpd_base_url
                 [{
                     'manifest_url': 'http://unknown/manifest.mpd',
                     'ext': 'mp4',
@@ -733,15 +735,61 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'width': 1920,
                     'height': 1080,
                 }]
+            ), (
+                # https://github.com/ytdl-org/youtube-dl/issues/20346
+                # Media considered unfragmented even though it contains
+                # Initialization tag
+                'unfragmented',
+                'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',  # mpd_url
+                'https://v.redd.it/hw1x7rcg7zl21',  # mpd_base_url
+                [{
+                    'url': 'https://v.redd.it/hw1x7rcg7zl21/audio',
+                    'manifest_url': 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',
+                    'ext': 'm4a',
+                    'format_id': 'AUDIO-1',
+                    'format_note': 'DASH audio',
+                    'container': 'm4a_dash',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 129.87,
+                    'asr': 48000,
+
+                }, {
+                    'url': 'https://v.redd.it/hw1x7rcg7zl21/DASH_240',
+                    'manifest_url': 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'VIDEO-2',
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d401e',
+                    'tbr': 608.0,
+                    'width': 240,
+                    'height': 240,
+                    'fps': 30,
+                }, {
+                    'url': 'https://v.redd.it/hw1x7rcg7zl21/DASH_360',
+                    'manifest_url': 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'VIDEO-1',
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d401e',
+                    'tbr': 804.261,
+                    'width': 360,
+                    'height': 360,
+                    'fps': 30,
+                }]
             )
         ]
 
-        for mpd_file, mpd_url, expected_formats in _TEST_CASES:
+        for mpd_file, mpd_url, mpd_base_url, expected_formats in _TEST_CASES:
             with io.open('./test/testdata/mpd/%s.mpd' % mpd_file,
                          mode='r', encoding='utf-8') as f:
                 formats = self.ie._parse_mpd_formats(
                     compat_etree_fromstring(f.read().encode('utf-8')),
-                    mpd_url=mpd_url)
+                    mpd_base_url=mpd_base_url, mpd_url=mpd_url)
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
 
diff --git a/test/testdata/mpd/unfragmented.mpd b/test/testdata/mpd/unfragmented.mpd
new file mode 100644
index 000000000..5a3720be7
--- /dev/null
+++ b/test/testdata/mpd/unfragmented.mpd
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<MPD mediaPresentationDuration="PT54.915S" minBufferTime="PT1.500S" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" type="static" xmlns="urn:mpeg:dash:schema:mpd:2011">
+    <Period duration="PT54.915S">
+        <AdaptationSet segmentAlignment="true" subsegmentAlignment="true" subsegmentStartsWithSAP="1">
+            <Representation bandwidth="804261" codecs="avc1.4d401e" frameRate="30" height="360" id="VIDEO-1" mimeType="video/mp4" startWithSAP="1" width="360">
+                <BaseURL>DASH_360</BaseURL>
+                <SegmentBase indexRange="915-1114" indexRangeExact="true">
+                    <Initialization range="0-914"/>
+                </SegmentBase>
+            </Representation>
+            <Representation bandwidth="608000" codecs="avc1.4d401e" frameRate="30" height="240" id="VIDEO-2" mimeType="video/mp4" startWithSAP="1" width="240">
+                <BaseURL>DASH_240</BaseURL>
+                <SegmentBase indexRange="913-1112" indexRangeExact="true">
+                    <Initialization range="0-912"/>
+                </SegmentBase>
+            </Representation>
+        </AdaptationSet>
+        <AdaptationSet>
+            <Representation audioSamplingRate="48000" bandwidth="129870" codecs="mp4a.40.2" id="AUDIO-1" mimeType="audio/mp4" startWithSAP="1">
+                <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+                <BaseURL>audio</BaseURL>
+                <SegmentBase indexRange="832-1007" indexRangeExact="true">
+                    <Initialization range="0-831"/>
+                </SegmentBase>
+            </Representation>
+        </AdaptationSet>
+    </Period>
+</MPD>

From ffbd1368df48932e9171e6c8a7a4958430000b13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 07:07:56 +0700
Subject: [PATCH 487/510] [update] Hide update URLs behind redirect

---
 youtube_dl/update.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index a14265c1b..002ea7f33 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -31,7 +31,7 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose, opener):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = 'https://ytdl-org.github.io/youtube-dl/update/'
+    UPDATE_URL = 'https://yt-dl.org/update/'
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

From 77d95677b7ab4a9840ef142b14627b07a9a31120 Mon Sep 17 00:00:00 2001
From: utlasidyo <42189435+utlasidyo@users.noreply.github.com>
Date: Sun, 17 Mar 2019 07:15:15 +0700
Subject: [PATCH 488/510] [youtube] Add support for invidiou.sh (#20309)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0bc6cc7a7..886fc1591 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -351,7 +351,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             (?:www\.)?hooktube\.com/|
                             (?:www\.)?yourepeat\.com/|
                             tube\.majestyc\.net/|
-                            (?:www\.)?invidio\.us/|
+                            (?:(?:www|dev)\.)?invidio\.us/|
+                            (?:www\.)?invidiou\.sh/|
                             (?:www\.)?invidious\.snopyta\.org/|
                             (?:www\.)?invidious\.kabi\.tk/|
                             (?:www\.)?vid\.wxzm\.sx/|

From e5cfb779ea0814971324d693e02f8c4d4ca78600 Mon Sep 17 00:00:00 2001
From: Austin de Coup-Crank <austindcc@gmail.com>
Date: Sat, 16 Mar 2019 17:18:54 -0700
Subject: [PATCH 489/510] [ciscolive] Add support for new URL schema (closes
 #20320, #20351)

---
 youtube_dl/extractor/ciscolive.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ciscolive.py b/youtube_dl/extractor/ciscolive.py
index c99b6ee58..da404e4dc 100644
--- a/youtube_dl/extractor/ciscolive.py
+++ b/youtube_dl/extractor/ciscolive.py
@@ -65,8 +65,8 @@ class CiscoLiveBaseIE(InfoExtractor):
 
 
 class CiscoLiveSessionIE(CiscoLiveBaseIE):
-    _VALID_URL = r'https?://ciscolive\.cisco\.com/on-demand-library/\??[^#]*#/session/(?P<id>[^/?&]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?ciscolive(?:\.cisco)?\.com/[^#]*#/session/(?P<id>[^/?&]+)'
+    _TESTS = [{
         'url': 'https://ciscolive.cisco.com/on-demand-library/?#/session/1423353499155001FoSs',
         'md5': 'c98acf395ed9c9f766941c70f5352e22',
         'info_dict': {
@@ -79,7 +79,13 @@ class CiscoLiveSessionIE(CiscoLiveBaseIE):
             'uploader_id': '5647924234001',
             'location': '16B Mezz.',
         },
-    }
+    }, {
+        'url': 'https://www.ciscolive.com/global/on-demand-library.html?search.event=ciscoliveemea2019#/session/15361595531500013WOU',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.ciscolive.com/global/on-demand-library.html?#/session/1490051371645001kNaS',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         rf_id = self._match_id(url)
@@ -88,7 +94,7 @@ class CiscoLiveSessionIE(CiscoLiveBaseIE):
 
 
 class CiscoLiveSearchIE(CiscoLiveBaseIE):
-    _VALID_URL = r'https?://ciscolive\.cisco\.com/on-demand-library/'
+    _VALID_URL = r'https?://(?:www\.)?ciscolive(?:\.cisco)?\.com/(?:global/)?on-demand-library(?:\.html|/)'
     _TESTS = [{
         'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
         'info_dict': {
@@ -98,6 +104,9 @@ class CiscoLiveSearchIE(CiscoLiveBaseIE):
     }, {
         'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ciscolive.com/global/on-demand-library.html?search.technicallevel=scpsSkillLevel_aintroductory&search.event=ciscoliveemea2019&search.technology=scpsTechnology_dataCenter&search.focus=scpsSessionFocus_bestPractices#/',
+        'only_matching': True,
     }]
 
     @classmethod

From 0dc41787af0fb011f01d88c1f2ecb686a8416df5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 09:07:47 +0700
Subject: [PATCH 490/510] [utils] Introduce parse_bitrate

---
 test/test_utils.py  | 8 ++++++++
 youtube_dl/utils.py | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 409482c3b..acd994bd7 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -55,6 +55,7 @@ from youtube_dl.utils import (
     parse_count,
     parse_iso8601,
     parse_resolution,
+    parse_bitrate,
     pkcs1pad,
     read_batch_urls,
     sanitize_filename,
@@ -1030,6 +1031,13 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_resolution('4k'), {'height': 2160})
         self.assertEqual(parse_resolution('8K'), {'height': 4320})
 
+    def test_parse_bitrate(self):
+        self.assertEqual(parse_bitrate(None), None)
+        self.assertEqual(parse_bitrate(''), None)
+        self.assertEqual(parse_bitrate('300kbps'), 300)
+        self.assertEqual(parse_bitrate('1500kbps'), 1500)
+        self.assertEqual(parse_bitrate('300 kbps'), 300)
+
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 630dab8eb..f2726a579 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1798,6 +1798,14 @@ def parse_resolution(s):
     return {}
 
 
+def parse_bitrate(s):
+    if not isinstance(s, compat_str):
+        return
+    mobj = re.search(r'\b(\d+)\s*kbps', s)
+    if mobj:
+        return int(mobj.group(1))
+
+
 def month_by_name(name, lang='en'):
     """ Return the number of a month by (locale-independently) English name """
 

From d493f15c1158abd817e191ff830fd5481b1ed42d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 09:09:32 +0700
Subject: [PATCH 491/510] [extractor/common] Improve HTML5 entries extraction
 and add some realworld tests

---
 test/test_InfoExtractor.py     | 178 +++++++++++++++++++++++++++++++++
 youtube_dl/extractor/common.py |  43 ++++++--
 2 files changed, 214 insertions(+), 7 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index d23d94349..71f6608fe 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -107,6 +107,184 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertRaises(ExtractorError, self.ie._download_json, uri, None)
         self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
 
+    def test_parse_html5_media_entries(self):
+        # from https://www.r18.com/
+        # with kpbs in label
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://www.r18.com/',
+                r'''
+                <video id="samplevideo_amateur" class="js-samplevideo video-js vjs-default-skin vjs-big-play-centered" controls preload="auto" width="400" height="225" poster="//pics.r18.com/digital/amateur/mgmr105/mgmr105jp.jpg">
+                    <source id="video_source" src="https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_sm_w.mp4" type="video/mp4"  res="240" label="300kbps">
+                    <source id="video_source" src="https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dm_w.mp4" type="video/mp4"  res="480" label="1000kbps">
+                    <source id="video_source" src="https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dmb_w.mp4" type="video/mp4"  res="740" label="1500kbps">
+                    <p>Your browser does not support the video tag.</p>
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'url': 'https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_sm_w.mp4',
+                    'ext': 'mp4',
+                    'format_id': '300kbps',
+                    'height': 240,
+                    'tbr': 300,
+                }, {
+                    'url': 'https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dm_w.mp4',
+                    'ext': 'mp4',
+                    'format_id': '1000kbps',
+                    'height': 480,
+                    'tbr': 1000,
+                }, {
+                    'url': 'https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dmb_w.mp4',
+                    'ext': 'mp4',
+                    'format_id': '1500kbps',
+                    'height': 740,
+                    'tbr': 1500,
+                }],
+                'thumbnail': '//pics.r18.com/digital/amateur/mgmr105/mgmr105jp.jpg'
+            })
+
+        # from https://www.csfd.cz/
+        # with width and height
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://www.csfd.cz/',
+                r'''
+                <video width="770" height="328" preload="none" controls  poster="https://img.csfd.cz/files/images/film/video/preview/163/344/163344118_748d20.png?h360" >
+                    <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327358_eac647.mp4" type="video/mp4" width="640" height="360">
+                    <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327360_3d2646.mp4" type="video/mp4" width="1280" height="720">
+                    <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327356_91f258.mp4" type="video/mp4" width="1920" height="1080">
+                    <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327359_962b4a.webm" type="video/webm" width="640" height="360">
+                    <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327361_6feee0.webm" type="video/webm" width="1280" height="720">
+                    <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327357_8ab472.webm" type="video/webm" width="1920" height="1080">
+                    <track src="https://video.csfd.cz/files/subtitles/163/344/163344115_4c388b.srt" type="text/x-srt" kind="subtitles" srclang="cs" label="cs">
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327358_eac647.mp4',
+                    'ext': 'mp4',
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327360_3d2646.mp4',
+                    'ext': 'mp4',
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327356_91f258.mp4',
+                    'ext': 'mp4',
+                    'width': 1920,
+                    'height': 1080,
+                }, {
+                    'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327359_962b4a.webm',
+                    'ext': 'webm',
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327361_6feee0.webm',
+                    'ext': 'webm',
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327357_8ab472.webm',
+                    'ext': 'webm',
+                    'width': 1920,
+                    'height': 1080,
+                }],
+                'subtitles': {
+                    'cs': [{'url': 'https://video.csfd.cz/files/subtitles/163/344/163344115_4c388b.srt'}]
+                },
+                'thumbnail': 'https://img.csfd.cz/files/images/film/video/preview/163/344/163344118_748d20.png?h360'
+            })
+
+        # from https://tamasha.com/v/Kkdjw
+        # with height in label
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://tamasha.com/v/Kkdjw',
+                r'''
+                <video crossorigin="anonymous">
+                        <source src="https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4" type="video/mp4" label="AUTO" res="0"/>
+                                <source src="https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4" type="video/mp4"
+                                        label="240p" res="240"/>
+                                <source src="https://s-v2.tamasha.com/statics/videos_file/20/00/Kkdjw_200041c66f657fc967db464d156eafbc1ed9fe6f_n_144.mp4" type="video/mp4"
+                                        label="144p" res="144"/>
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'url': 'https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4',
+                }, {
+                    'url': 'https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4',
+                    'ext': 'mp4',
+                    'format_id': '240p',
+                    'height': 240,
+                }, {
+                    'url': 'https://s-v2.tamasha.com/statics/videos_file/20/00/Kkdjw_200041c66f657fc967db464d156eafbc1ed9fe6f_n_144.mp4',
+                    'ext': 'mp4',
+                    'format_id': '144p',
+                    'height': 144,
+                }]
+            })
+
+        # from https://www.directvnow.com
+        # with data-src
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://www.directvnow.com',
+                r'''
+                <video id="vid1" class="header--video-masked active" muted playsinline>
+                    <source data-src="https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4" type="video/mp4" />
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'ext': 'mp4',
+                    'url': 'https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4',
+                }]
+            })
+
+        # from https://www.directvnow.com
+        # with data-src
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://www.directvnow.com',
+                r'''
+                <video id="vid1" class="header--video-masked active" muted playsinline>
+                    <source data-src="https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4" type="video/mp4" />
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'url': 'https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4',
+                    'ext': 'mp4',
+                }]
+            })
+
+        # from https://www.klarna.com/uk/
+        # with data-video-src
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://www.directvnow.com',
+                r'''
+                <video loop autoplay muted class="responsive-video block-kl__video video-on-medium">
+                    <source src="" data-video-desktop data-video-src="https://www.klarna.com/uk/wp-content/uploads/sites/11/2019/01/KL062_Smooth3_0_DogWalking_5s_920x080_.mp4" type="video/mp4" />
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'url': 'https://www.klarna.com/uk/wp-content/uploads/sites/11/2019/01/KL062_Smooth3_0_DogWalking_5s_920x080_.mp4',
+                    'ext': 'mp4',
+                }],
+            })
+
     def test_extract_jwplayer_data_realworld(self):
         # from http://www.suffolk.edu/sjc/
         expect_dict(
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c291bc1df..0889288f0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -44,6 +44,7 @@ from ..utils import (
     compiled_regex_type,
     determine_ext,
     determine_protocol,
+    dict_get,
     error_to_compat_str,
     ExtractorError,
     extract_attributes,
@@ -56,13 +57,16 @@ from ..utils import (
     JSON_LD_RE,
     mimetype2ext,
     orderedSet,
+    parse_bitrate,
     parse_codecs,
     parse_duration,
     parse_iso8601,
     parse_m3u8_attributes,
+    parse_resolution,
     RegexNotFoundError,
     sanitized_Request,
     sanitize_filename,
+    str_or_none,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -2481,18 +2485,43 @@ class InfoExtractor(object):
             media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
             if media_content:
                 for source_tag in re.findall(r'<source[^>]+>', media_content):
-                    source_attributes = extract_attributes(source_tag)
-                    src = source_attributes.get('src')
+                    s_attr = extract_attributes(source_tag)
+                    # data-video-src and data-src are non standard but seen
+                    # several times in the wild
+                    src = dict_get(s_attr, ('src', 'data-video-src', 'data-src'))
                     if not src:
                         continue
-                    f = parse_content_type(source_attributes.get('type'))
+                    f = parse_content_type(s_attr.get('type'))
                     is_plain_url, formats = _media_formats(src, media_type, f)
                     if is_plain_url:
-                        # res attribute is not standard but seen several times
-                        # in the wild
+                        # width, height, res, label and title attributes are
+                        # all not standard but seen several times in the wild
+                        labels = [
+                            s_attr.get(lbl)
+                            for lbl in ('label', 'title')
+                            if str_or_none(s_attr.get(lbl))
+                        ]
+                        width = int_or_none(s_attr.get('width'))
+                        height = (int_or_none(s_attr.get('height')) or
+                                  int_or_none(s_attr.get('res')))
+                        if not width or not height:
+                            for lbl in labels:
+                                resolution = parse_resolution(lbl)
+                                if not resolution:
+                                    continue
+                                width = width or resolution.get('width')
+                                height = height or resolution.get('height')
+                        for lbl in labels:
+                            tbr = parse_bitrate(lbl)
+                            if tbr:
+                                break
+                        else:
+                            tbr = None
                         f.update({
-                            'height': int_or_none(source_attributes.get('res')),
-                            'format_id': source_attributes.get('label'),
+                            'width': width,
+                            'height': height,
+                            'tbr': tbr,
+                            'format_id': s_attr.get('label') or s_attr.get('title'),
                         })
                         f.update(formats[0])
                         media_info['formats'].append(f)

From eba3a2f9ef52abcc0d71afeb2162581cc8958367 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 09:15:24 +0700
Subject: [PATCH 492/510] [anysex] Remove extractor (closes #19279)

---
 youtube_dl/extractor/anysex.py     | 61 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 62 deletions(-)
 delete mode 100644 youtube_dl/extractor/anysex.py

diff --git a/youtube_dl/extractor/anysex.py b/youtube_dl/extractor/anysex.py
deleted file mode 100644
index ad86d6e58..000000000
--- a/youtube_dl/extractor/anysex.py
+++ /dev/null
@@ -1,61 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-    int_or_none,
-)
-
-
-class AnySexIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?anysex\.com/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://anysex.com/156592/',
-        'md5': '023e9fbb7f7987f5529a394c34ad3d3d',
-        'info_dict': {
-            'id': '156592',
-            'ext': 'mp4',
-            'title': 'Busty and sexy blondie in her bikini strips for you',
-            'description': 'md5:de9e418178e2931c10b62966474e1383',
-            'categories': ['Erotic'],
-            'duration': 270,
-            'age_limit': 18,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, video_id)
-
-        video_url = self._html_search_regex(r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
-
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        description = self._html_search_regex(
-            r'<div class="description"[^>]*>([^<]+)</div>', webpage, 'description', fatal=False)
-        thumbnail = self._html_search_regex(
-            r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
-
-        categories = re.findall(
-            r'<a href="http://anysex\.com/categories/[^"]+" title="[^"]*">([^<]+)</a>', webpage)
-
-        duration = parse_duration(self._search_regex(
-            r'<b>Duration:</b> (?:<q itemprop="duration">)?(\d+:\d+)', webpage, 'duration', fatal=False))
-        view_count = int_or_none(self._html_search_regex(
-            r'<b>Views:</b> (\d+)', webpage, 'view count', fatal=False))
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'categories': categories,
-            'duration': duration,
-            'view_count': view_count,
-            'age_limit': 18,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9a0459de8..e27a09509 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -39,7 +39,6 @@ from .amcnetworks import AMCNetworksIE
 from .americastestkitchen import AmericasTestKitchenIE
 from .animeondemand import AnimeOnDemandIE
 from .anvato import AnvatoIE
-from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
 from .aliexpress import AliExpressLiveIE

From 81dada0b4ba156812a89f0896579830cccd21fa3 Mon Sep 17 00:00:00 2001
From: Tyler Szabo <tyler.szabo@gmail.com>
Date: Sat, 16 Mar 2019 19:37:29 -0700
Subject: [PATCH 493/510] [cbc:watch] Add support for gem.cbc.ca (closes
 #20251, #20359)

---
 youtube_dl/extractor/cbc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 43f95c739..751a3a8f2 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -360,7 +360,7 @@ class CBCWatchVideoIE(CBCWatchBaseIE):
 
 class CBCWatchIE(CBCWatchBaseIE):
     IE_NAME = 'cbc.ca:watch'
-    _VALID_URL = r'https?://watch\.cbc\.ca/(?:[^/]+/)+(?P<id>[0-9a-f-]+)'
+    _VALID_URL = r'https?://(?:gem|watch)\.cbc\.ca/(?:[^/]+/)+(?P<id>[0-9a-f-]+)'
     _TESTS = [{
         # geo-restricted to Canada, bypassable
         'url': 'http://watch.cbc.ca/doc-zone/season-6/customer-disservice/38e815a-009e3ab12e4',
@@ -386,6 +386,9 @@ class CBCWatchIE(CBCWatchBaseIE):
             'description': 'Arthur, the sweetest 8-year-old aardvark, and his pals solve all kinds of problems with humour, kindness and teamwork.',
         },
         'playlist_mincount': 30,
+    }, {
+        'url': 'https://gem.cbc.ca/media/this-hour-has-22-minutes/season-26/episode-20/38e815a-0108c6c6a42',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ddff25c5d1f21c323cc37c08faa11ea98baef622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 09:41:16 +0700
Subject: [PATCH 494/510] [extractors] Remove superfluous whitespace

---
 youtube_dl/extractor/extractors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e27a09509..f32f22a5a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1400,7 +1400,7 @@ from .webofstories import (
     WebOfStoriesPlaylistIE,
 )
 from .weibo import (
-    WeiboIE, 
+    WeiboIE,
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE

From 0146c6cde6959a52287c26c99ce159128832246c Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Sun, 17 Mar 2019 03:57:02 +0100
Subject: [PATCH 495/510] [orf:radio] Extract series (#20012)

---
 youtube_dl/extractor/orf.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index d432e3449..499be0029 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -176,7 +176,8 @@ class ORFRadioIE(InfoExtractor):
                 'description': subtitle,
                 'duration': (info['end'] - info['start']) / 1000,
                 'timestamp': info['start'] / 1000,
-                'ext': 'mp3'
+                'ext': 'mp3',
+                'series': data.get('programTitle')
             }
 
         entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]

From a63782b58117d651bc2d32edada2e795dfd9eb41 Mon Sep 17 00:00:00 2001
From: wolfy1339 <4595477+wolfy1339@users.noreply.github.com>
Date: Sun, 17 Mar 2019 03:20:21 -0400
Subject: [PATCH 496/510] [corus] Add support for bigbrothercanada.ca (#20357)

---
 youtube_dl/extractor/corus.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/corus.py b/youtube_dl/extractor/corus.py
index 807a29eea..a1b251804 100644
--- a/youtube_dl/extractor/corus.py
+++ b/youtube_dl/extractor/corus.py
@@ -13,9 +13,9 @@ class CorusIE(ThePlatformFeedIE):
                         (?:www\.)?
                         (?P<domain>
                             (?:globaltv|etcanada)\.com|
-                            (?:hgtv|foodnetwork|slice|history|showcase)\.ca
+                            (?:hgtv|foodnetwork|slice|history|showcase|bigbrothercanada)\.ca
                         )
-                        /(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))
+                        /(?:video/(?:[^/]+/)?|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))
                         (?P<id>\d+)
                     '''
     _TESTS = [{
@@ -42,6 +42,12 @@ class CorusIE(ThePlatformFeedIE):
     }, {
         'url': 'http://www.showcase.ca/eyewitness/video/eyewitness++106/video.html?v=955070531919&p=1&s=da#video',
         'only_matching': True,
+    }, {
+        'url': 'http://www.bigbrothercanada.ca/video/1457812035894/',
+        'only_matching': True
+    }, {
+        'url': 'https://www.bigbrothercanada.ca/video/big-brother-canada-704/1457812035894/',
+        'only_matching': True
     }]
 
     _TP_FEEDS = {
@@ -73,6 +79,10 @@ class CorusIE(ThePlatformFeedIE):
             'feed_id': '9H6qyshBZU3E',
             'account_id': 2414426607,
         },
+        'bigbrothercanada': {
+            'feed_id': 'ChQqrem0lNUp',
+            'account_id': 2269680845,
+        },
     }
 
     def _real_extract(self, url):

From 034f5fb5ee37e9edf90f7e82f48af985cdd17901 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 14:34:48 +0700
Subject: [PATCH 497/510] [radiocanada:audiovideo] Fix typo

---
 youtube_dl/extractor/radiocanada.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 2e3f1f58f..a28b1a24c 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -147,7 +147,7 @@ class RadioCanadaIE(InfoExtractor):
 
 
 class RadioCanadaAudioVideoIE(InfoExtractor):
-    'radiocanada:audiovideo'
+    IE_NAME = 'radiocanada:audiovideo'
     _VALID_URL = r'https?://ici\.radio-canada\.ca/([^/]+/)*media-(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://ici.radio-canada.ca/audio-video/media-7527184/barack-obama-au-vietnam',

From 04988b55b54bd7bd803d43a56f87e4728158890a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 15:34:21 +0700
Subject: [PATCH 498/510] [openload] Improve embed detection

---
 youtube_dl/extractor/openload.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index bae7c7ee7..cc323e5e2 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -243,18 +243,16 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
+    _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space))'
     _VALID_URL = r'''(?x)
                     https?://
                         (?P<host>
                             (?:www\.)?
-                            (?:
-                                openload\.(?:co|io|link|pw)|
-                                oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)
-                            )
+                            %s
                         )/
                         (?:f|embed)/
                         (?P<id>[a-zA-Z0-9-_]+)
-                    '''
+                    ''' % _DOMAINS
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -359,8 +357,8 @@ class OpenloadIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
-            r'<iframe[^>]+src=["\']((?:https?://)?(?:openload\.(?:co|io)|oload\.tv)/embed/[a-zA-Z0-9-_]+)',
-            webpage)
+            r'<iframe[^>]+src=["\']((?:https?://)?%s/embed/[a-zA-Z0-9-_]+)'
+            % OpenloadIE._DOMAINS, webpage)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 8428fdccf2dad7667cf53db449282b56dbbfa3ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Mar 2019 17:33:07 +0700
Subject: [PATCH 499/510] [yandexvideo] Add extractor

---
 youtube_dl/extractor/extractors.py  |  3 +-
 youtube_dl/extractor/yandexvideo.py | 90 +++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/yandexvideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f32f22a5a..8e7a5bf41 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1444,12 +1444,13 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
+from .yandexdisk import YandexDiskIE
 from .yandexmusic import (
     YandexMusicTrackIE,
     YandexMusicAlbumIE,
     YandexMusicPlaylistIE,
 )
-from .yandexdisk import YandexDiskIE
+from .yandexvideo import YandexVideoIE
 from .yapfiles import YapFilesIE
 from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
diff --git a/youtube_dl/extractor/yandexvideo.py b/youtube_dl/extractor/yandexvideo.py
new file mode 100644
index 000000000..940c24af3
--- /dev/null
+++ b/youtube_dl/extractor/yandexvideo.py
@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    url_or_none,
+)
+
+
+class YandexVideoIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            yandex\.ru(?:/portal/(?:video|efir))?/?\?.*?stream_id=|
+                            frontend\.vh\.yandex\.ru/player/
+                        )
+                        (?P<id>[\da-f]+)
+                    '''
+    _TESTS = [{
+        'url': 'https://yandex.ru/portal/video?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
+        'md5': '33955d7ae052f15853dc41f35f17581c',
+        'info_dict': {
+            'id': '4dbb262b4fe5cf15a215de4f34eee34d',
+            'ext': 'mp4',
+            'title': 'В Нью-Йорке баржи и теплоход оторвались от причала и расплылись по Гудзону',
+            'description': '',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 0,
+            'duration': 30,
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'https://yandex.ru/portal/efir?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374&from=morda',
+        'only_matching': True,
+    }, {
+        'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
+        'only_matching': True,
+    }, {
+        'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
+        'only_matching': True,
+    }, {
+        # vod-episode, series episode
+        'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
+        'only_matching': True,
+    }, {
+        # episode, sports
+        'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        content = self._download_json(
+            'https://frontend.vh.yandex.ru/v22/player/%s.json' % video_id,
+            video_id, query={
+                'stream_options': 'hires',
+                'disable_trackings': 1,
+            })['content']
+
+        m3u8_url = url_or_none(content.get('content_url')) or url_or_none(
+            content['streams'][0]['url'])
+        title = content.get('title') or content.get('computed_title')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+        self._sort_formats(formats)
+
+        description = content.get('description')
+        thumbnail = content.get('thumbnail')
+        timestamp = (int_or_none(content.get('release_date')) or
+                     int_or_none(content.get('release_date_ut')) or
+                     int_or_none(content.get('start_time')))
+        duration = int_or_none(content.get('duration'))
+        series = content.get('program_title')
+        age_limit = int_or_none(content.get('restriction_age'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'series': series,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From 2ed2ebdb36c678ac47b5bec1d4d2eaeda471d0f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Mar 2019 01:33:37 +0700
Subject: [PATCH 500/510] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index eda94ad33..0ab376d83 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,37 @@
+version <unreleased>
+
+Core
+* [extractor/common] Improve HTML5 entries extraction
++ [utils] Introduce parse_bitrate
+* [update] Hide update URLs behind redirect
+* [extractor/common] Fix url meta field for unfragmented DASH formats (#20346)
+
+Extractors
++ [yandexvideo] Add extractor
+* [openload] Improve embed detection
++ [corus] Add support for bigbrothercanada.ca (#20357)
++ [orf:radio] Extract series (#20012)
++ [cbc:watch] Add support for gem.cbc.ca (#20251, #20359)
+- [anysex] Remove extractor (#19279)
++ [ciscolive] Add support for new URL schema (#20320, #20351)
++ [youtube] Add support for invidiou.sh (#20309)
+- [anitube] Remove extractor (#20334)
+- [ruleporn] Remove extractor (#15344, #20324)
+* [npr] Fix extraction (#10793, #13440)
+* [biqle] Fix extraction (#11471, #15313)
+* [viddler] Modernize
+* [moevideo] Fix extraction
+* [primesharetv] Remove extractor
+* [hypem] Modernize and extract more metadata (#15320)
+* [veoh] Fix extraction
+* [escapist] Modernize
+- [videomega] Remove extractor (#10108)
++ [beeg] Add support for beeg.porn (#20306)
+* [vimeo:review] Improve config url extraction and extract original format
+  (#20305)
+* [fox] Detect geo restriction and authentication errors (#20208)
+
+
 version 2019.03.09
 
 Core

From 0a8e251b35db6a05949c61ee4e56b678697f08a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Mar 2019 01:36:41 +0700
Subject: [PATCH 501/510] release 2019.03.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 8 ++------
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 911e912a4..0decf19a1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.09*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.03.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.03.18*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.03.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2019.03.09
+[debug] youtube-dl version 2019.03.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0ab376d83..d0e3a6088 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2019.03.18
 
 Core
 * [extractor/common] Improve HTML5 entries extraction
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d8a8d7ede..a3d4447a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,9 +44,7 @@
  - **AmericasTestKitchen**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimeOnDemand**
- - **anitube.se**
  - **Anvato**
- - **AnySex**
  - **APA**
  - **Aparat**
  - **AppleConnect**
@@ -698,7 +696,6 @@
  - **PornoXO**
  - **PornTube**
  - **PressTV**
- - **PrimeShareTV**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **puhutv**
@@ -718,7 +715,7 @@
  - **radio.de**
  - **radiobremen**
  - **radiocanada**
- - **RadioCanadaAudioVideo**
+ - **radiocanada:audiovideo**
  - **radiofrance**
  - **RadioJavan**
  - **Rai**
@@ -765,7 +762,6 @@
  - **RTVS**
  - **Rudo**
  - **RUHD**
- - **RulePorn**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
  - **rutube:embed**: Rutube embedded videos
@@ -1010,7 +1006,6 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **VideoMega**
  - **videomore**
  - **videomore:season**
  - **videomore:video**
@@ -1127,6 +1122,7 @@
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
+ - **YandexVideo**
  - **YapFiles**
  - **YesJapan**
  - **yinyuetai:video**: 音悦Tai
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f72fee57f..5e86bc4d5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2019.03.09'
+__version__ = '2019.03.18'

From c4580580f58d004a0e4e9a5d90b0a44cdd69c7e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Mar 2019 22:39:35 +0700
Subject: [PATCH 502/510] [svtplay] Update API endpoint (closes #20430)

---
 youtube_dl/extractor/svt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 0901c3163..7aa1b5919 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -185,7 +185,7 @@ class SVTPlayIE(SVTPlayBaseIE):
 
     def _extract_by_video_id(self, video_id, webpage=None):
         data = self._download_json(
-            'https://api.svt.se/videoplayer-api/video/%s' % video_id,
+            'https://api.svt.se/video/%s' % video_id,
             video_id, headers=self.geo_verification_headers())
         info_dict = self._extract_video(data, video_id)
         if not info_dict.get('title'):

From 050afa60c6685f0b0aff68d943d48b3b6b9c85a0 Mon Sep 17 00:00:00 2001
From: Jesse de Zwart <jessedezwart@live.nl>
Date: Thu, 21 Mar 2019 16:55:03 +0100
Subject: [PATCH 503/510] Check for valid --min-sleep-interval when
 --max-sleep-interval is specified

---
 youtube_dl/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 94788d936..9d4859bcf 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -166,6 +166,8 @@ def _real_main(argv=None):
     if opts.max_sleep_interval is not None:
         if opts.max_sleep_interval < 0:
             parser.error('max sleep interval must be positive or 0')
+        if opts.sleep_interval is None:
+            parser.error('min sleep interval must be specified, use --min-sleep-interval')
         if opts.max_sleep_interval < opts.sleep_interval:
             parser.error('max sleep interval must be greater than or equal to min sleep interval')
     else:

From 5e1271c56dbf3f96fbf928ccd3facf5fc70493b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Mar 2019 01:08:54 +0700
Subject: [PATCH 504/510] [utils] Improve int_or_none and float_or_none
 (#20403)

---
 test/test_utils.py  | 17 +++++++++++++++++
 youtube_dl/utils.py |  4 ++--
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index acd994bd7..ca6d832a4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -33,11 +33,13 @@ from youtube_dl.utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
+    float_or_none,
     get_element_by_class,
     get_element_by_attribute,
     get_elements_by_class,
     get_elements_by_attribute,
     InAdvancePagedList,
+    int_or_none,
     intlist_to_bytes,
     is_html,
     js_to_json,
@@ -468,6 +470,21 @@ class TestUtil(unittest.TestCase):
             shell_quote(args),
             """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if compat_os_name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
 
+    def test_float_or_none(self):
+        self.assertEqual(float_or_none('42.42'), 42.42)
+        self.assertEqual(float_or_none('42'), 42.0)
+        self.assertEqual(float_or_none(''), None)
+        self.assertEqual(float_or_none(None), None)
+        self.assertEqual(float_or_none([]), None)
+        self.assertEqual(float_or_none(set()), None)
+
+    def test_int_or_none(self):
+        self.assertEqual(int_or_none('42'), 42)
+        self.assertEqual(int_or_none(''), None)
+        self.assertEqual(int_or_none(None), None)
+        self.assertEqual(int_or_none([]), None)
+        self.assertEqual(int_or_none(set()), None)
+
     def test_str_to_int(self):
         self.assertEqual(str_to_int('123,456'), 123456)
         self.assertEqual(str_to_int('123.456'), 123456)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f2726a579..71713f63a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1922,7 +1922,7 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
         return default
     try:
         return int(v) * invscale // scale
-    except ValueError:
+    except (ValueError, TypeError):
         return default
 
 
@@ -1943,7 +1943,7 @@ def float_or_none(v, scale=1, invscale=1, default=None):
         return default
     try:
         return float(v) * invscale / scale
-    except ValueError:
+    except (ValueError, TypeError):
         return default
 
 

From b8526c78f9a8a9101c5fd365f07c12e7b53c5ed9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Mar 2019 01:09:33 +0700
Subject: [PATCH 505/510] [pornhub] Add support for DASH formats (closes
 #20403)

---
 youtube_dl/extractor/pornhub.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3a474c179..bf8f0be88 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -14,6 +14,7 @@ from ..compat import (
 )
 from .openload import PhantomJSwrapper
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     orderedSet,
@@ -275,6 +276,10 @@ class PornHubIE(PornHubBaseIE):
                     r'/(\d{6}/\d{2})/', video_url, 'upload data', default=None)
                 if upload_date:
                     upload_date = upload_date.replace('/', '')
+            if determine_ext(video_url) == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    video_url, video_id, mpd_id='dash', fatal=False))
+                continue
             tbr = None
             mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
             if mobj:

From 8cb10807ed6c6aa8e0f316b3f1a31a91df46abdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Mar 2019 21:43:50 +0700
Subject: [PATCH 506/510] [npo] Improve DRM detection

---
 youtube_dl/extractor/npo.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index ad62f8ec6..e525ad928 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -181,10 +181,7 @@ class NPOIE(NPOBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        try:
-            return self._get_info(url, video_id)
-        except ExtractorError:
-            return self._get_old_info(video_id)
+        return self._get_info(url, video_id) or self._get_old_info(video_id)
 
     def _get_info(self, url, video_id):
         token = self._download_json(
@@ -206,6 +203,7 @@ class NPOIE(NPOBaseIE):
 
         player_token = player['token']
 
+        drm = False
         format_urls = set()
         formats = []
         for profile in ('hls', 'dash-widevine', 'dash-playready', 'smooth'):
@@ -227,7 +225,8 @@ class NPOIE(NPOBaseIE):
             if not stream_url or stream_url in format_urls:
                 continue
             format_urls.add(stream_url)
-            if stream.get('protection') is not None:
+            if stream.get('protection') is not None or stream.get('keySystemOptions') is not None:
+                drm = True
                 continue
             stream_type = stream.get('type')
             stream_ext = determine_ext(stream_url)
@@ -246,6 +245,11 @@ class NPOIE(NPOBaseIE):
                     'url': stream_url,
                 })
 
+        if not formats:
+            if drm:
+                raise ExtractorError('This video is DRM protected.', expected=True)
+            return
+
         self._sort_formats(formats)
 
         info = {

From de74ef83b786daad1b3df738734ca81ce3344667 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 27 Mar 2019 18:01:51 +0100
Subject: [PATCH 507/510] [cwtv] fix episode number extraction(closes #20461)

---
 youtube_dl/extractor/cwtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index f9bd535f6..73382431b 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -79,7 +79,7 @@ class CWTVIE(InfoExtractor):
         season = str_or_none(video_data.get('season'))
         episode = str_or_none(video_data.get('episode'))
         if episode and season:
-            episode = episode.lstrip(season)
+            episode = episode[len(season):]
 
         return {
             '_type': 'url_transparent',

From b27a71e66c1eb31887aa34eb8dbadb22b91bf716 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 27 Mar 2019 18:29:24 +0100
Subject: [PATCH 508/510] [ina] improve extraction

---
 youtube_dl/extractor/ina.py | 64 +++++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ina.py b/youtube_dl/extractor/ina.py
index 9544ff9d4..5744a52b8 100644
--- a/youtube_dl/extractor/ina.py
+++ b/youtube_dl/extractor/ina.py
@@ -1,36 +1,72 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    strip_or_none,
+    xpath_attr,
+    xpath_text,
+)
 
 
 class InaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>[A-Z0-9_]+)'
+    _TESTS = [{
         'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
         'md5': 'a667021bf2b41f8dc6049479d9bb38a3',
         'info_dict': {
             'id': 'I12055569',
             'ext': 'mp4',
             'title': 'François Hollande "Je crois que c\'est clair"',
+            'description': 'md5:3f09eb072a06cb286b8f7e4f77109663',
         }
-    }
+    }, {
+        'url': 'https://www.ina.fr/video/S806544_001/don-d-organes-des-avancees-mais-d-importants-besoins-video.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id = self._match_id(url)
+        info_doc = self._download_xml(
+            'http://player.ina.fr/notices/%s.mrss' % video_id, video_id)
+        item = info_doc.find('channel/item')
+        title = xpath_text(item, 'title', fatal=True)
+        media_ns_xpath = lambda x: self._xpath_ns(x, 'http://search.yahoo.com/mrss/')
+        content = item.find(media_ns_xpath('content'))
 
-        video_id = mobj.group('id')
-        mrss_url = 'http://player.ina.fr/notices/%s.mrss' % video_id
-        info_doc = self._download_xml(mrss_url, video_id)
+        get_furl = lambda x: xpath_attr(content, media_ns_xpath(x), 'url')
+        formats = []
+        for q, w, h in (('bq', 400, 300), ('mq', 512, 384), ('hq', 768, 576)):
+            q_url = get_furl(q)
+            if not q_url:
+                continue
+            formats.append({
+                'format_id': q,
+                'url': q_url,
+                'width': w,
+                'height': h,
+            })
+        if not formats:
+            formats = [{
+                'url': get_furl('player') or content.attrib['url'],
+            }]
 
-        self.report_extraction(video_id)
-
-        video_url = info_doc.find('.//{http://search.yahoo.com/mrss/}player').attrib['url']
+        thumbnails = []
+        for thumbnail in content.findall(media_ns_xpath('thumbnail')):
+            thumbnail_url = thumbnail.get('url')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'height': int_or_none(thumbnail.get('height')),
+                'width': int_or_none(thumbnail.get('width')),
+            })
 
         return {
             'id': video_id,
-            'url': video_url,
-            'title': info_doc.find('.//title').text,
+            'formats': formats,
+            'title': title,
+            'description': strip_or_none(xpath_text(item, 'description')),
+            'thumbnails': thumbnails,
         }

From c4c888697ec6b1576f01273a333a3a08ffbf6831 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 27 Mar 2019 18:49:29 +0100
Subject: [PATCH 509/510] [ina] add support for audio URLs

---
 youtube_dl/extractor/ina.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ina.py b/youtube_dl/extractor/ina.py
index 5744a52b8..12695af27 100644
--- a/youtube_dl/extractor/ina.py
+++ b/youtube_dl/extractor/ina.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
     strip_or_none,
     xpath_attr,
@@ -11,7 +12,7 @@ from ..utils import (
 
 
 class InaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>[A-Z0-9_]+)'
+    _VALID_URL = r'https?://(?:www\.)?ina\.fr/(?:video|audio)/(?P<id>[A-Z0-9_]+)'
     _TESTS = [{
         'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
         'md5': 'a667021bf2b41f8dc6049479d9bb38a3',
@@ -24,6 +25,12 @@ class InaIE(InfoExtractor):
     }, {
         'url': 'https://www.ina.fr/video/S806544_001/don-d-organes-des-avancees-mais-d-importants-besoins-video.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ina.fr/audio/P16173408',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.ina.fr/video/P16173408-video.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -48,8 +55,12 @@ class InaIE(InfoExtractor):
                 'height': h,
             })
         if not formats:
+            furl = get_furl('player') or content.attrib['url']
+            ext = determine_ext(furl)
             formats = [{
-                'url': get_furl('player') or content.attrib['url'],
+                'url': furl,
+                'vcodec': 'none' if ext == 'mp3' else None,
+                'ext': ext,
             }]
 
         thumbnails = []

From 99fe3300707464a926ab1f62925cbbad90c3bde2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 28 Mar 2019 16:55:57 +0100
Subject: [PATCH 510/510] [teamtreehouse] Add new extractor(closes #9836)

---
 youtube_dl/extractor/extractors.py    |   1 +
 youtube_dl/extractor/teamtreehouse.py | 140 ++++++++++++++++++++++++++
 2 files changed, 141 insertions(+)
 create mode 100644 youtube_dl/extractor/teamtreehouse.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8e7a5bf41..e79ffca85 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1114,6 +1114,7 @@ from .teachertube import (
 )
 from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
+from .teamtreehouse import TeamTreeHouseIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
 from .tele5 import Tele5IE
diff --git a/youtube_dl/extractor/teamtreehouse.py b/youtube_dl/extractor/teamtreehouse.py
new file mode 100644
index 000000000..d347e97ef
--- /dev/null
+++ b/youtube_dl/extractor/teamtreehouse.py
@@ -0,0 +1,140 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    determine_ext,
+    ExtractorError,
+    float_or_none,
+    get_element_by_class,
+    get_element_by_id,
+    parse_duration,
+    remove_end,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class TeamTreeHouseIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?teamtreehouse\.com/library/(?P<id>[^/]+)'
+    _TESTS = [{
+        # Course
+        'url': 'https://teamtreehouse.com/library/introduction-to-user-authentication-in-php',
+        'info_dict': {
+            'id': 'introduction-to-user-authentication-in-php',
+            'title': 'Introduction to User Authentication in PHP',
+            'description': 'md5:405d7b4287a159b27ddf30ca72b5b053',
+        },
+        'playlist_mincount': 24,
+    }, {
+        # WorkShop
+        'url': 'https://teamtreehouse.com/library/deploying-a-react-app',
+        'info_dict': {
+            'id': 'deploying-a-react-app',
+            'title': 'Deploying a React App',
+            'description': 'md5:10a82e3ddff18c14ac13581c9b8e5921',
+        },
+        'playlist_mincount': 4,
+    }, {
+        # Video
+        'url': 'https://teamtreehouse.com/library/application-overview-2',
+        'info_dict': {
+            'id': 'application-overview-2',
+            'ext': 'mp4',
+            'title': 'Application Overview',
+            'description': 'md5:4b0a234385c27140a4378de5f1e15127',
+        },
+        'expected_warnings': ['This is just a preview'],
+    }]
+    _NETRC_MACHINE = 'teamtreehouse'
+
+    def _real_initialize(self):
+        email, password = self._get_login_info()
+        if email is None:
+            return
+
+        signin_page = self._download_webpage(
+            'https://teamtreehouse.com/signin',
+            None, 'Downloading signin page')
+        data = self._form_hidden_inputs('new_user_session', signin_page)
+        data.update({
+            'user_session[email]': email,
+            'user_session[password]': password,
+        })
+        error_message = get_element_by_class('error-message', self._download_webpage(
+            'https://teamtreehouse.com/person_session',
+            None, 'Logging in', data=urlencode_postdata(data)))
+        if error_message:
+            raise ExtractorError(clean_html(error_message), expected=True)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        title = self._html_search_meta(['og:title', 'twitter:title'], webpage)
+        description = self._html_search_meta(
+            ['description', 'og:description', 'twitter:description'], webpage)
+        entries = self._parse_html5_media_entries(url, webpage, display_id)
+        if entries:
+            info = entries[0]
+
+            for subtitles in info.get('subtitles', {}).values():
+                for subtitle in subtitles:
+                    subtitle['ext'] = determine_ext(subtitle['url'], 'srt')
+
+            is_preview = 'data-preview="true"' in webpage
+            if is_preview:
+                self.report_warning(
+                    'This is just a preview. You need to be signed in with a Basic account to download the entire video.', display_id)
+                duration = 30
+            else:
+                duration = float_or_none(self._search_regex(
+                    r'data-duration="(\d+)"', webpage, 'duration'), 1000)
+                if not duration:
+                    duration = parse_duration(get_element_by_id(
+                        'video-duration', webpage))
+
+            info.update({
+                'id': display_id,
+                'title': title,
+                'description': description,
+                'duration': duration,
+            })
+            return info
+        else:
+            def extract_urls(html, extract_info=None):
+                for path in re.findall(r'<a[^>]+href="([^"]+)"', html):
+                    page_url = urljoin(url, path)
+                    entry = {
+                        '_type': 'url_transparent',
+                        'id': self._match_id(page_url),
+                        'url': page_url,
+                        'id_key': self.ie_key(),
+                    }
+                    if extract_info:
+                        entry.update(extract_info)
+                    entries.append(entry)
+
+            workshop_videos = self._search_regex(
+                r'(?s)<ul[^>]+id="workshop-videos"[^>]*>(.+?)</ul>',
+                webpage, 'workshop videos', default=None)
+            if workshop_videos:
+                extract_urls(workshop_videos)
+            else:
+                stages_path = self._search_regex(
+                    r'(?s)<div[^>]+id="syllabus-stages"[^>]+data-url="([^"]+)"',
+                    webpage, 'stages path')
+                if stages_path:
+                    stages_page = self._download_webpage(
+                        urljoin(url, stages_path), display_id, 'Downloading stages page')
+                    for chapter_number, (chapter, steps_list) in enumerate(re.findall(r'(?s)<h2[^>]*>\s*(.+?)\s*</h2>.+?<ul[^>]*>(.+?)</ul>', stages_page), 1):
+                        extract_urls(steps_list, {
+                            'chapter': chapter,
+                            'chapter_number': chapter_number,
+                        })
+                    title = remove_end(title, ' Course')
+
+            return self.playlist_result(
+                entries, display_id, title, description)